OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix(2478): track eval baseline (#2582)
Sathvik Lakamsani23b1533fada1
diff --git a/.github/workflows/eval-retrieval.yml b/.github/workflows/eval-retrieval.ymlindex c2f5a2649..5e3c7d694 100644--- a/.github/workflows/eval-retrieval.yml+++ b/.github/workflows/eval-retrieval.yml@@ -234,7 +234,7 @@ jobs:cd "$GITHUB_WORKSPACE"git config user.name "github-actions[bot]"git config user.email "41898282+github-actions[bot]@users.noreply.github.com"- git add scripts/eval/retrieval/baseline.json+ git add -f scripts/eval/retrieval/baseline.jsongit commit -m "chore: update the retrieval eval baseline"git push origin "HEAD:refs/heads/${REF_NAME}"diff --git a/.gitignore b/.gitignoreindex 5656f8589..05ccdf56a 100644--- a/.gitignore+++ b/.gitignore@@ -24,6 +24,7 @@ wheels/!src/tui/_assets/flows/*.json!src/tui/_assets/flows/components/*.json!frontend/*.json+!scripts/eval/retrieval/baseline.json.DS_Storelogsdiff --git a/scripts/eval/retrieval/baseline.json b/scripts/eval/retrieval/baseline.jsonnew file mode 100644index 000000000..219c6590f--- /dev/null+++ b/scripts/eval/retrieval/baseline.json@@ -0,0 +1,12195 @@+{+ "schema_version": 1,+ "run": {+ "run_id": "eval-49debd2f",+ "timestamp": "2026-11-05T18:03:57Z",+ "duration_s": 85.9,+ "target_url": "http://localhost:3000",+ "sdk_version": "0.4.0.dev0",+ "git_commit": "a7076dc551c4b940b63119b9c242665fa927f880",+ "corpus_files": 21,+ "corpus_hash": "9a346ec3274599771b53573f708fca382c23a27b19d102a84210377df06ab73a",+ "query_count": 83,+ "passage_questions": 77,+ "queries_hash": "a76d3448549d17810642d5a5206a56f54b8e8f605b43def0d3791941ba9f31bf",+ "k_values": [+ 1,+ 3,+ 5,+ 10+ ],+ "primary_k": 5,+ "fetch_limit": 50,+ "score_threshold": 0.0+ },+ "settings": {+ "embedding_provider": "azure",+ "embedding_model": "text-embedding-3-small",+ "chunk_size": 1000,+ "chunk_overlap": 200,+ "table_structure": true,+ "ocr": false,+ "picture_descriptions": false+ },+ "settings_note": null,+ "metrics": {+ "overall": {+ "recall@1": 0.6265060240963856,+ "recall@3": 0.8493975903614458,+ "recall@5": 0.9096385542168675,+ "recall@10": 0.963855421686747,+ "mrr": 0.7938898450946643,+ "ndcg@1": 0.7108433734939759,+ "ndcg@3": 0.7762517657398734,+ "ndcg@5": 0.8030453462731483,+ "ndcg@10": 0.8211616182694974,+ "passage_hit@1": 0.5324675324675324,+ "passage_hit@3": 0.7012987012987013,+ "passage_hit@5": 0.7922077922077922,+ "passage_hit@10": 0.8701298701298701,+ "passage_mrr": 0.6384012790334507+ },+ "by_tag": {+ "exact-token": {+ "count": 13,+ "passage_count": 13,+ "recall@1": 0.7307692307692307,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "mrr": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.9876493521766443,+ "ndcg@5": 0.9876493521766443,+ "ndcg@10": 0.9876493521766443,+ "passage_hit@1": 0.8461538461538461,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_mrr": 0.9230769230769231+ },+ "keyword": {+ "count": 7,+ "passage_count": 6,+ "recall@1": 0.5714285714285714,+ "recall@3": 0.9285714285714286,+ "recall@5": 0.9285714285714286,+ "recall@10": 1.0,+ "mrr": 0.8333333333333333,+ "ndcg@1": 0.7142857142857143,+ "ndcg@3": 0.8205824209052738,+ "ndcg@5": 0.8205824209052738,+ "ndcg@10": 0.8459023190172805,+ "passage_hit@1": 0.6666666666666666,+ "passage_hit@3": 0.8333333333333334,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_mrr": 0.7638888888888888+ },+ "multi-doc": {+ "count": 16,+ "passage_count": 11,+ "recall@1": 0.4375,+ "recall@3": 0.71875,+ "recall@5": 0.84375,+ "recall@10": 0.9375,+ "mrr": 0.903125,+ "ndcg@1": 0.875,+ "ndcg@3": 0.7440735963827292,+ "ndcg@5": 0.8050524251877592,+ "ndcg@10": 0.8413163264616205,+ "passage_hit@1": 0.7272727272727273,+ "passage_hit@3": 0.9090909090909091,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_mrr": 0.8257575757575758+ },+ "semantic": {+ "count": 63,+ "passage_count": 58,+ "recall@1": 0.6111111111111112,+ "recall@3": 0.8095238095238095,+ "recall@5": 0.8888888888888888,+ "recall@10": 0.9523809523809523,+ "mrr": 0.7469765684051398,+ "ndcg@1": 0.6507936507936508,+ "ndcg@3": 0.7277044132027969,+ "ndcg@5": 0.7630038923180636,+ "ndcg@10": 0.7840580397611262,+ "passage_hit@1": 0.4482758620689655,+ "passage_hit@3": 0.6206896551724138,+ "passage_hit@5": 0.7241379310344828,+ "passage_hit@10": 0.8275862068965517,+ "passage_mrr": 0.5616131922800408+ }+ }+ },+ "queries": [+ {+ "id": "key-rotate",+ "query": "How do I swap out an API key without causing downtime?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "api-keys.md"+ ],+ "expected_text": "overlap window",+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "api-keys.md",+ "rank": 1,+ "score": 4.485182,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.1399813,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 3.9088736,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 4,+ "score": 3.9022071,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.8871171,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 6,+ "score": 3.7825556,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 7,+ "score": 3.5549948,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 3.3526278,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 3.1453872,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 3.0133064,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 2.9531522,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 12,+ "score": 2.9356976,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 2.8147604,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 2.790321,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 2.6110516,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 16,+ "score": 2.5505896,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.3984993,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "api-keys.md": 1+ },+ "snippet_found": true,+ "passage_rank": 10,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.1+ }+ },+ {+ "id": "key-leaked",+ "query": "One of our keys was pushed to a public repo, what should we do right now?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "api-keys.md"+ ],+ "expected_text": "Revoke immediately",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "api-keys.md",+ "rank": 1,+ "score": 4.5577087,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 2,+ "score": 4.4239907,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 3.8773596,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 4,+ "score": 3.7586002,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 5,+ "score": 3.7468228,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 6,+ "score": 3.6312208,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 7,+ "score": 3.373435,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 8,+ "score": 3.268697,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 9,+ "score": 3.2684705,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 3.224593,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 3.1721559,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 3.1261327,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 13,+ "score": 3.0540926,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 14,+ "score": 3.023003,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.9438276,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 16,+ "score": 2.9110684,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 2.8642788,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 2.65653,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 19,+ "score": 2.5132947,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "api-keys.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "key-scopes",+ "query": "What scopes can I give an API key?",+ "tags": [+ "keyword"+ ],+ "expected_files": [+ "api-keys.md"+ ],+ "expected_text": "destinations:manage",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "api-keys.md",+ "rank": 1,+ "score": 5.5974355,+ "snippet_found": true+ },+ {+ "filename": "error-codes.md",+ "rank": 2,+ "score": 5.0317492,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 3,+ "score": 4.3466816,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 3.7902362,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.6450343,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 6,+ "score": 3.5535414,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 7,+ "score": 3.533179,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 8,+ "score": 3.477205,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 9,+ "score": 3.3471954,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 3.334623,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 3.179077,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 12,+ "score": 3.147795,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 13,+ "score": 3.0607226,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 14,+ "score": 2.8921738,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.697281,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 2.5704966,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 2.479469,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 18,+ "score": 2.3684938,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 19,+ "score": 2.3103514,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "api-keys.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "key-prefix-ci",+ "query": "What is a key starting with rwk_ci_ used for?",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "api-keys.md"+ ],+ "expected_text": "Automation key",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "api-keys.md",+ "rank": 1,+ "score": 5.3789306,+ "snippet_found": true+ },+ {+ "filename": "cli-reference.md",+ "rank": 2,+ "score": 4.635419,+ "snippet_found": true+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 3.814974,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 4,+ "score": 3.5494313,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 5,+ "score": 3.0389695,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 6,+ "score": 2.91465,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 2.8971639,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 2.8961177,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 9,+ "score": 2.875177,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.8743634,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 2.7834806,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 2.7593927,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 13,+ "score": 2.5931182,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 14,+ "score": 2.5388367,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 2.5265174,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 2.4976048,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.3593998,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 18,+ "score": 2.2960348,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 19,+ "score": 2.2928317,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 20,+ "score": 2.171982,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "api-keys.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "sig-verify",+ "query": "How can my server confirm that a webhook really came from Relaywise and was not forged?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "signing-secrets.md"+ ],+ "expected_text": "HMAC-SHA256",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "signing-secrets.md",+ "rank": 1,+ "score": 5.4620786,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 2,+ "score": 5.1428657,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 5.107025,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 4,+ "score": 4.8653693,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 5,+ "score": 4.459428,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 6,+ "score": 4.323626,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 4.092972,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 8,+ "score": 4.032093,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 9,+ "score": 3.9983268,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 10,+ "score": 3.9548411,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 11,+ "score": 3.8796291,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 12,+ "score": 3.8189056,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 3.8055673,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 14,+ "score": 3.7612507,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 15,+ "score": 3.748525,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 3.7323694,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 3.6424484,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 18,+ "score": 3.5377412,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 19,+ "score": 3.460004,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 20,+ "score": 3.3449996,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "signing-secrets.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "sig-roll",+ "query": "How do I change the secret used to sign deliveries, and how long do the old and new ones both work?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "signing-secrets.md"+ ],+ "expected_text": "72 hours",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "audit-log.md",+ "rank": 1,+ "score": 8.1549225,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 7.6038017,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 3,+ "score": 7.401187,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 6.34908,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 6.135564,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 6,+ "score": 5.6227202,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 7,+ "score": 5.285435,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 5.0490365,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 9,+ "score": 4.9172583,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 10,+ "score": 4.916431,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 11,+ "score": 4.8609548,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 12,+ "score": 4.823242,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 13,+ "score": 4.7858067,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 14,+ "score": 4.7696395,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 4.65145,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 4.4600706,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 17,+ "score": 4.159475,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 18,+ "score": 3.9943762,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 19,+ "score": 3.8904157,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 20,+ "score": 3.7599213,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "signing-secrets.md": 3+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "sig-max-age",+ "query": "How old can a signed request be before I should refuse it?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "signing-secrets.md"+ ],+ "expected_text": "five minutes",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "signing-secrets.md",+ "rank": 1,+ "score": 5.4995303,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 5.3574557,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 4.620363,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 4,+ "score": 4.157878,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 5,+ "score": 3.9991243,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 6,+ "score": 3.7961216,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 3.7712054,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 3.6903996,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 9,+ "score": 3.672656,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 3.3198228,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 11,+ "score": 3.3114727,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 3.3018017,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 13,+ "score": 3.2214346,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 3.219668,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 2.914604,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 2.8591857,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.8397179,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 18,+ "score": 2.7246382,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 19,+ "score": 2.6846876,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 20,+ "score": 2.6837616,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 21,+ "score": 2.5916595,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "signing-secrets.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "sig-whsec",+ "query": "Where do I find the whsec_ value for a destination?",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "signing-secrets.md"+ ],+ "expected_text": "Reveal",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "signing-secrets.md",+ "rank": 1,+ "score": 6.0598135,+ "snippet_found": true+ },+ {+ "filename": "configuration-reference.md",+ "rank": 2,+ "score": 4.84408,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 4.648713,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.378025,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.803798,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 6,+ "score": 3.4391446,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 3.3304148,+ "snippet_found": true+ },+ {+ "filename": "regions.md",+ "rank": 8,+ "score": 3.3192945,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 9,+ "score": 2.9882765,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 10,+ "score": 2.8431144,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 11,+ "score": 2.8061593,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 2.6899495,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 13,+ "score": 2.6079462,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 14,+ "score": 2.5567536,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 2.555109,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 2.4441526,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 17,+ "score": 2.355443,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 18,+ "score": 2.2208116,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 19,+ "score": 2.1636539,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "signing-secrets.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "sso-setup",+ "query": "How do we let staff log in to the console with their company accounts?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "single-sign-on.md"+ ],+ "expected_text": "identity provider",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "audit-log.md",+ "rank": 1,+ "score": 4.916114,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 2,+ "score": 4.913085,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 4.8817015,+ "snippet_found": true+ },+ {+ "filename": "cli-reference.md",+ "rank": 4,+ "score": 4.7266417,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 5,+ "score": 4.4032316,+ "snippet_found": true+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 6,+ "score": 4.382999,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 7,+ "score": 4.0702477,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 8,+ "score": 4.055292,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 3.9624317,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 10,+ "score": 3.7526836,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 11,+ "score": 3.4365666,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 12,+ "score": 3.0746837,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 13,+ "score": 3.007635,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 14,+ "score": 2.8946323,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 2.8831072,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 2.7998972,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.716467,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 18,+ "score": 2.7119622,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 19,+ "score": 2.5842686,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 20,+ "score": 2.5467687,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "single-sign-on.md": 5+ },+ "snippet_found": true,+ "passage_rank": 6,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.2,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.38685280723454163,+ "ndcg@10": 0.38685280723454163,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.16666666666666666+ }+ },+ {+ "id": "sso-lockout",+ "query": "If our identity provider goes down, can anyone still get into the console?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "single-sign-on.md"+ ],+ "expected_text": "break-glass",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "single-sign-on.md",+ "rank": 1,+ "score": 7.240963,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 3.965464,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 3.9560423,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 4,+ "score": 3.5496902,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 5,+ "score": 3.2137072,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 6,+ "score": 3.2129083,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 7,+ "score": 3.2106323,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.1655471,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 9,+ "score": 3.0810347,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 10,+ "score": 3.0189486,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 2.8896675,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 2.8698277,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 2.6514459,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 14,+ "score": 2.6178923,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 2.6146202,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 2.5582738,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 17,+ "score": 2.5566497,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 2.5499167,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 19,+ "score": 2.1304684,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "single-sign-on.md": 1+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "sso-offboarding",+ "query": "How is someone's access removed automatically when they leave the company?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "single-sign-on.md"+ ],+ "expected_text": "SCIM",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "single-sign-on.md",+ "rank": 1,+ "score": 5.3100443,+ "snippet_found": true+ },+ {+ "filename": "signing-secrets.md",+ "rank": 2,+ "score": 4.564142,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 3,+ "score": 4.112088,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 4,+ "score": 4.080737,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 5,+ "score": 3.9498796,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 6,+ "score": 3.8103957,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 7,+ "score": 3.748855,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 3.6031716,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 9,+ "score": 3.5195658,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 3.4794078,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 11,+ "score": 3.4275222,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 12,+ "score": 3.338546,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 13,+ "score": 3.278238,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 3.2054431,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 15,+ "score": 3.070385,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 16,+ "score": 2.9049387,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 17,+ "score": 2.4249506,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 18,+ "score": 2.311882,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 19,+ "score": 2.2950606,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 20,+ "score": 2.2608783,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "single-sign-on.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "session-length",+ "query": "How long before the console logs me out?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "single-sign-on.md",+ "configuration-reference.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "network-security.md",+ "rank": 1,+ "score": 4.718949,+ "snippet_found": null+ },+ {+ "filename": "audit-log.md",+ "rank": 2,+ "score": 4.631812,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 3,+ "score": 4.393179,+ "snippet_found": null+ },+ {+ "filename": "transformations.md",+ "rank": 4,+ "score": 3.361267,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 3.2693686,+ "snippet_found": null+ },+ {+ "filename": "single-sign-on.md",+ "rank": 6,+ "score": 3.2635205,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 7,+ "score": 3.2346013,+ "snippet_found": null+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 8,+ "score": 3.1757994,+ "snippet_found": null+ },+ {+ "filename": "billing-plans.md",+ "rank": 9,+ "score": 3.057241,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 10,+ "score": 2.9434013,+ "snippet_found": null+ },+ {+ "filename": "rate-limits.md",+ "rank": 11,+ "score": 2.8661394,+ "snippet_found": null+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 12,+ "score": 2.8531485,+ "snippet_found": null+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 2.7566361,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 14,+ "score": 2.6207955,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 15,+ "score": 2.4442549,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 16,+ "score": 2.4371514,+ "snippet_found": null+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 2.3959312,+ "snippet_found": null+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 2.392912,+ "snippet_found": null+ },+ {+ "filename": "api-keys.md",+ "rank": 19,+ "score": 2.146281,+ "snippet_found": null+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 20,+ "score": 1.9371018,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "single-sign-on.md": 6,+ "configuration-reference.md": 5+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.5,+ "recall@10": 1.0,+ "reciprocal_rank": 0.2,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.23719771276929622,+ "ndcg@10": 0.45560514958746035+ }+ },+ {+ "id": "dlq-purge-who",+ "query": "Who is allowed to empty a dead-letter queue?",+ "tags": [+ "keyword",+ "multi-doc"+ ],+ "expected_files": [+ "roles-and-permissions.md",+ "dead-letter-queue.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 5.936977,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 2,+ "score": 5.469982,+ "snippet_found": null+ },+ {+ "filename": "api-keys.md",+ "rank": 3,+ "score": 4.6652865,+ "snippet_found": null+ },+ {+ "filename": "troubleshooting.md",+ "rank": 4,+ "score": 4.4582715,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 5,+ "score": 4.1814966,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 6,+ "score": 4.075127,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 7,+ "score": 4.028639,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 8,+ "score": 3.9006016,+ "snippet_found": null+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 3.8686628,+ "snippet_found": null+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 10,+ "score": 3.857781,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 11,+ "score": 3.807433,+ "snippet_found": null+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 3.4361827,+ "snippet_found": null+ },+ {+ "filename": "network-security.md",+ "rank": 13,+ "score": 2.9324346,+ "snippet_found": null+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 2.883803,+ "snippet_found": null+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 2.7358198,+ "snippet_found": null+ },+ {+ "filename": "rate-limits.md",+ "rank": 16,+ "score": 2.616968,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 17,+ "score": 2.392273,+ "snippet_found": null+ },+ {+ "filename": "event-filtering.md",+ "rank": 18,+ "score": 2.3337693,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "roles-and-permissions.md": 10,+ "dead-letter-queue.md": 1+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 0.5,+ "recall@5": 0.5,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.6131471927654584,+ "ndcg@5": 0.6131471927654584,+ "ndcg@10": 0.7903864795495061+ }+ },+ {+ "id": "viewer-payloads",+ "query": "Can a read-only user see the contents of events?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "roles-and-permissions.md"+ ],+ "expected_text": "Payload access",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "roles-and-permissions.md",+ "rank": 1,+ "score": 4.9205885,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.8499303,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 3,+ "score": 4.828339,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.6417646,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 5,+ "score": 4.445083,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 6,+ "score": 4.2635155,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 7,+ "score": 3.988793,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 8,+ "score": 3.8740609,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 3.845269,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 3.5629725,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 3.4642334,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 3.359555,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 13,+ "score": 3.3335032,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 14,+ "score": 3.3266916,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 3.1584954,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 16,+ "score": 3.0837433,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 17,+ "score": 3.0490007,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 2.998888,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 19,+ "score": 2.7927902,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "roles-and-permissions.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "invite-expiry",+ "query": "How long does a team invitation link stay valid?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "roles-and-permissions.md"+ ],+ "expected_text": "valid for 7 days",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "roles-and-permissions.md",+ "rank": 1,+ "score": 5.6239557,+ "snippet_found": true+ },+ {+ "filename": "audit-log.md",+ "rank": 2,+ "score": 4.895487,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 3,+ "score": 4.1866374,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 4.109663,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 5,+ "score": 4.0150146,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 6,+ "score": 3.691551,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 3.5175457,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 3.4534285,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 9,+ "score": 3.2989433,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 3.2002692,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 11,+ "score": 2.9563212,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 12,+ "score": 2.823382,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 2.8055763,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 14,+ "score": 2.7686763,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 2.7495055,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 16,+ "score": 2.7170486,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 17,+ "score": 2.7069454,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 2.6110837,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 19,+ "score": 2.4631553,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 20,+ "score": 2.0309317,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 21,+ "score": 2.0022187,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "roles-and-permissions.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "ownership-transfer",+ "query": "How do I hand a workspace over to a colleague?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "roles-and-permissions.md"+ ],+ "expected_text": "last Owner",+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 4.235373,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 2,+ "score": 4.2214937,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 3,+ "score": 4.050256,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 4,+ "score": 3.4798121,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 5,+ "score": 3.3237166,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 3.3111908,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 7,+ "score": 3.0920124,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 8,+ "score": 3.0649781,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 2.9621043,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.6565669,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 11,+ "score": 2.6309865,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 2.5766845,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 2.551534,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 2.524885,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 15,+ "score": 2.5160358,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 16,+ "score": 2.360861,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 17,+ "score": 2.2255254,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "roles-and-permissions.md": 4+ },+ "snippet_found": true,+ "passage_rank": 5,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.25,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.43067655807339306,+ "ndcg@10": 0.43067655807339306,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.2+ }+ },+ {+ "id": "retry-schedule",+ "query": "How many times will a failed webhook be attempted, and over how long?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "delivery-retries.md"+ ],+ "expected_text": "27 hours",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "delivery-retries.md",+ "rank": 1,+ "score": 5.953916,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 5.9165845,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 3,+ "score": 5.629542,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 5.3382273,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 5,+ "score": 5.2244463,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 6,+ "score": 4.7950573,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 4.6740932,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 8,+ "score": 4.488102,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 4.288582,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 10,+ "score": 4.247769,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 11,+ "score": 4.176223,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 12,+ "score": 4.0876427,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 3.7292528,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 14,+ "score": 3.6605527,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 15,+ "score": 3.4551523,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 16,+ "score": 3.4263456,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 17,+ "score": 3.2239738,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 2.5783687,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 19,+ "score": 2.4560182,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 1+ },+ "snippet_found": true,+ "passage_rank": 6,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.16666666666666666+ }+ },+ {+ "id": "cfg-max-attempts",+ "query": "What is the default for delivery.max_attempts?",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "delivery-retries.md",+ "configuration-reference.md"+ ],+ "expected_text": "delivery.max_attempts",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "delivery-retries.md",+ "rank": 1,+ "score": 4.100553,+ "snippet_found": true+ },+ {+ "filename": "configuration-reference.md",+ "rank": 2,+ "score": 3.990864,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 3,+ "score": 2.4812531,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 2.3984542,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 5,+ "score": 2.262816,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 6,+ "score": 2.2063332,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 7,+ "score": 2.1881351,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 8,+ "score": 2.1569738,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 9,+ "score": 2.0785315,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.0601966,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 11,+ "score": 2.0537386,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 12,+ "score": 2.0393662,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 13,+ "score": 2.037712,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 14,+ "score": 1.9448432,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 15,+ "score": 1.8971537,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 1.8890164,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 17,+ "score": 1.8565283,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 18,+ "score": 1.8449395,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 19,+ "score": 1.8378721,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 20,+ "score": 1.8362129,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 1,+ "configuration-reference.md": 2+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "circuit-breaker",+ "query": "What happens when most requests to one endpoint start failing?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "delivery-retries.md"+ ],+ "expected_text": "degraded",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 1,+ "score": 5.0543203,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 2,+ "score": 4.903905,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 4.752765,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.5258617,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 5,+ "score": 4.1057053,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 6,+ "score": 4.084237,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 7,+ "score": 3.9673247,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.947473,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 3.8320982,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 10,+ "score": 3.7889988,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 11,+ "score": 3.705308,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 12,+ "score": 3.6559482,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 13,+ "score": 3.598694,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 14,+ "score": 3.2583046,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 15,+ "score": 3.1484778,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 16,+ "score": 2.9071856,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 2.8332481,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 2.7670798,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 19,+ "score": 2.5437899,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 2+ },+ "snippet_found": true,+ "passage_rank": 2,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.5,+ "ndcg@1": 0.0,+ "ndcg@3": 0.6309297535714575,+ "ndcg@5": 0.6309297535714575,+ "ndcg@10": 0.6309297535714575,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.5+ }+ },+ {+ "id": "gone-410",+ "query": "What does Relaywise do if my endpoint answers 410 Gone?",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "delivery-retries.md",+ "dead-letter-queue.md"+ ],+ "expected_text": "410 Gone",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "delivery-retries.md",+ "rank": 1,+ "score": 5.6472993,+ "snippet_found": true+ },+ {+ "filename": "error-codes.md",+ "rank": 2,+ "score": 4.9761944,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 4.925442,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 4,+ "score": 4.794298,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 5,+ "score": 4.408581,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 6,+ "score": 4.156478,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 7,+ "score": 4.053644,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 3.842249,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 9,+ "score": 3.769952,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 10,+ "score": 3.6307647,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 3.591357,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 12,+ "score": 3.158509,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 13,+ "score": 3.0485291,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 14,+ "score": 2.950628,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.7675254,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 2.7243469,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.7232513,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 18,+ "score": 2.5460482,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 19,+ "score": 2.5377195,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 20,+ "score": 2.4653406,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 1,+ "dead-letter-queue.md": 3+ },+ "snippet_found": true,+ "passage_rank": 2,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.9197207891481876,+ "ndcg@5": 0.9197207891481876,+ "ndcg@10": 0.9197207891481876,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.5+ }+ },+ {+ "id": "duplicate-events",+ "query": "Why did my endpoint get the same event twice?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "delivery-retries.md",+ "troubleshooting.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 6.117975,+ "snippet_found": null+ },+ {+ "filename": "event-filtering.md",+ "rank": 2,+ "score": 5.701635,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 4.079763,+ "snippet_found": null+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 3.9514606,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 3.9501805,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 6,+ "score": 3.8345904,+ "snippet_found": null+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 3.5684028,+ "snippet_found": null+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 3.4548547,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 3.4442132,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 3.2959082,+ "snippet_found": null+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 3.278033,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 3.2729166,+ "snippet_found": null+ },+ {+ "filename": "transformations.md",+ "rank": 13,+ "score": 3.2519,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 3.0763547,+ "snippet_found": null+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 15,+ "score": 2.9944477,+ "snippet_found": null+ },+ {+ "filename": "network-security.md",+ "rank": 16,+ "score": 2.795557,+ "snippet_found": null+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 2.4639845,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 5,+ "troubleshooting.md": 1+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 0.5,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.6131471927654584,+ "ndcg@5": 0.8503449055347546,+ "ndcg@10": 0.8503449055347546+ }+ },+ {+ "id": "ordered-delivery",+ "query": "Can I make sure events for one customer are delivered in sequence?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "delivery-retries.md"+ ],+ "expected_text": "ordering key",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 5.0055847,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 2,+ "score": 4.8828163,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 3,+ "score": 4.750148,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 4.722822,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 5,+ "score": 4.638679,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 6,+ "score": 4.568696,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 4.3880367,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 4.3245573,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 9,+ "score": 4.2667556,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 10,+ "score": 4.13976,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 11,+ "score": 4.1162443,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 4.1093607,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 4.1089215,+ "snippet_found": true+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 4.0251784,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 3.751133,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 16,+ "score": 3.4920778,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 3.3043363,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 3.2784362,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 3+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "destination-auto-disabled",+ "query": "Why was my destination turned off without anyone touching it?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "delivery-retries.md",+ "troubleshooting.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.720254,+ "snippet_found": null+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 4.6855354,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 4.230507,+ "snippet_found": null+ },+ {+ "filename": "audit-log.md",+ "rank": 4,+ "score": 4.065977,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 3.9990711,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 6,+ "score": 3.6033638,+ "snippet_found": null+ },+ {+ "filename": "network-security.md",+ "rank": 7,+ "score": 3.3606205,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 3.3185124,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 9,+ "score": 3.0583892,+ "snippet_found": null+ },+ {+ "filename": "event-filtering.md",+ "rank": 10,+ "score": 2.7916946,+ "snippet_found": null+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 2.7792022,+ "snippet_found": null+ },+ {+ "filename": "signing-secrets.md",+ "rank": 12,+ "score": 2.6849184,+ "snippet_found": null+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 13,+ "score": 2.385123,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 14,+ "score": 2.2865496,+ "snippet_found": null+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 2.131452,+ "snippet_found": null+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.1214397,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 2.0443506,+ "snippet_found": null+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 1.9859804,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "delivery-retries.md": 5,+ "troubleshooting.md": 1+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 0.5,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.6131471927654584,+ "ndcg@5": 0.8503449055347546,+ "ndcg@10": 0.8503449055347546+ }+ },+ {+ "id": "dlq-what",+ "query": "Where do events go once every delivery attempt has been used up?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "dead-letter-queue.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 7.40966,+ "snippet_found": null+ },+ {+ "filename": "rate-limits.md",+ "rank": 2,+ "score": 6.4829597,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 3,+ "score": 6.2448034,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 4,+ "score": 5.8351073,+ "snippet_found": null+ },+ {+ "filename": "event-filtering.md",+ "rank": 5,+ "score": 5.679214,+ "snippet_found": null+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 5.4656425,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 5.4533553,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 8,+ "score": 5.332749,+ "snippet_found": null+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 4.9943905,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 4.968727,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 4.9405565,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 4.714102,+ "snippet_found": null+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 4.632801,+ "snippet_found": null+ },+ {+ "filename": "network-security.md",+ "rank": 14,+ "score": 4.599147,+ "snippet_found": null+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 4.52356,+ "snippet_found": null+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 4.458351,+ "snippet_found": null+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 17,+ "score": 4.3109264,+ "snippet_found": null+ },+ {+ "filename": "api-keys.md",+ "rank": 18,+ "score": 3.9744198,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "dead-letter-queue.md": 1+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0+ }+ },+ {+ "id": "dlq-retention-team",+ "query": "On the Team plan, how long are undeliverable events kept before they are removed?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "dead-letter-queue.md"+ ],+ "expected_text": "14 days",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "data-retention.md",+ "rank": 1,+ "score": 7.8197613,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 2,+ "score": 7.3345337,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 3,+ "score": 7.332274,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 6.697984,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 5.410752,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 6,+ "score": 5.0231695,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 4.924449,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 4.6471853,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 9,+ "score": 4.3576174,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 10,+ "score": 4.2551737,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 4.2262077,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 12,+ "score": 4.0809183,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 3.9892564,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 14,+ "score": 3.9197059,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 15,+ "score": 3.8503838,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 16,+ "score": 3.7696798,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 17,+ "score": 3.5610816,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 3.4350631,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "dead-letter-queue.md": 4+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.25,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.43067655807339306,+ "ndcg@10": 0.43067655807339306,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "dlq-replay-other-destination",+ "query": "How do I use rwctl dlq replay to resend failed events to a new endpoint?",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "dead-letter-queue.md"+ ],+ "expected_text": "dst_9x1aa",+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 8.870468,+ "snippet_found": true+ },+ {+ "filename": "cli-reference.md",+ "rank": 2,+ "score": 6.881461,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 5.751782,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 4,+ "score": 5.043284,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 5,+ "score": 4.7557015,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 4.6869016,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 7,+ "score": 4.359108,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 8,+ "score": 4.1611156,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 9,+ "score": 4.0299006,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 3.9948087,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 11,+ "score": 3.844854,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 3.5940793,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 13,+ "score": 3.408236,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 3.3590999,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 3.3468688,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 3.2550454,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 17,+ "score": 3.1481986,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "dead-letter-queue.md": 1+ },+ "snippet_found": true,+ "passage_rank": 2,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.5+ }+ },+ {+ "id": "replay-header",+ "query": "How can my endpoint tell that a delivery is a replay?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "dead-letter-queue.md"+ ],+ "expected_text": "Relaywise-Replay",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 5.8395176,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 5.7776,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 4.659917,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 4,+ "score": 4.5585823,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 4.535796,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 6,+ "score": 4.4854813,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 7,+ "score": 4.418273,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 4.399255,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 4.092454,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 10,+ "score": 3.7910876,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 11,+ "score": 3.6927078,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 3.5761201,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 3.4670155,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 14,+ "score": 3.373952,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 3.302986,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 3.2588396,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 3.145359,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 18,+ "score": 3.0399675,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "dead-letter-queue.md": 1+ },+ "snippet_found": true,+ "passage_rank": 10,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.1+ }+ },+ {+ "id": "ingest-rate",+ "query": "How fast am I allowed to send events on each plan?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "rate-limits.md"+ ],+ "expected_text": "events per second",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "rate-limits.md",+ "rank": 1,+ "score": 6.559215,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 2,+ "score": 6.377366,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 6.3756666,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 4,+ "score": 5.5507555,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 5,+ "score": 4.7628126,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 6,+ "score": 4.5803623,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 7,+ "score": 4.3698053,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 8,+ "score": 4.1771226,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 9,+ "score": 4.1309533,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 10,+ "score": 4.102806,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 11,+ "score": 3.8547676,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 12,+ "score": 3.605217,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 13,+ "score": 3.5623853,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 3.453558,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 15,+ "score": 3.3093355,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 16,+ "score": 3.2472394,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 17,+ "score": 3.2431312,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 3.1259098,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "rate-limits.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "err-4290",+ "query": "RW-4290",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "rate-limits.md",+ "error-codes.md"+ ],+ "expected_text": "RW-4290",+ "chunks_returned": 8,+ "unique_files_returned": 2,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 5.104968,+ "snippet_found": true+ },+ {+ "filename": "rate-limits.md",+ "rank": 2,+ "score": 4.61705,+ "snippet_found": true+ }+ ],+ "expected_ranks": {+ "rate-limits.md": 2,+ "error-codes.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "payload-max-size",+ "query": "What is the largest event body I can send?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "rate-limits.md"+ ],+ "expected_text": "256 KB",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.1728625,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 2,+ "score": 4.963256,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 4.2940288,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 3.8478184,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 5,+ "score": 3.7793765,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 6,+ "score": 3.7538767,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 3.7113836,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 8,+ "score": 3.647768,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 9,+ "score": 3.63453,+ "snippet_found": true+ },+ {+ "filename": "signing-secrets.md",+ "rank": 10,+ "score": 3.572057,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 3.561708,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 12,+ "score": 3.4531813,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 3.378756,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 14,+ "score": 3.3345623,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 3.2072341,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 16,+ "score": 3.0846722,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.9861572,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 18,+ "score": 2.791448,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 2.7770343,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "rate-limits.md": 9+ },+ "snippet_found": true,+ "passage_rank": 14,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.1111111111111111,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.0,+ "ndcg@10": 0.3010299956639812,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.07142857142857142+ }+ },+ {+ "id": "batch-size",+ "query": "How many events fit in one batch request?",+ "tags": [+ "keyword"+ ],+ "expected_files": [+ "rate-limits.md"+ ],+ "expected_text": "500 events",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "rate-limits.md",+ "rank": 1,+ "score": 6.440647,+ "snippet_found": true+ },+ {+ "filename": "event-filtering.md",+ "rank": 2,+ "score": 4.471184,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 3,+ "score": 4.471135,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 4.4559145,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 5,+ "score": 4.4337754,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 6,+ "score": 4.210376,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 7,+ "score": 4.1672397,+ "snippet_found": true+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.9377017,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 9,+ "score": 3.857754,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 10,+ "score": 3.7983751,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 3.741257,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 3.216155,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 13,+ "score": 3.2107887,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 14,+ "score": 3.144632,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 15,+ "score": 2.9834402,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 16,+ "score": 2.9600403,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 2.867922,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 18,+ "score": 2.8529837,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 19,+ "score": 2.5421212,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "rate-limits.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "err-3002",+ "query": "What does error RW-3002 mean?",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "error-codes.md"+ ],+ "expected_text": "TLS handshake failed",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 7.1374054,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 4.6004024,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 3,+ "score": 3.5720081,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 4,+ "score": 3.5708163,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 3.373531,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 3.3219764,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 7,+ "score": 3.270485,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 8,+ "score": 3.0856066,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 2.9129028,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 2.8195572,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 11,+ "score": 2.7496767,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 12,+ "score": 2.6049972,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 2.339275,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 14,+ "score": 2.285453,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 15,+ "score": 2.267887,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 16,+ "score": 2.1199887,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 1.9273345,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 1.7775464,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 1.6508594,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "error-codes.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "err-5004",+ "query": "RW-5004",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "error-codes.md",+ "transformations.md"+ ],+ "expected_text": "RW-5004",+ "chunks_returned": 8,+ "unique_files_returned": 2,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 7.083034,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 4.1704693,+ "snippet_found": true+ }+ ],+ "expected_ranks": {+ "error-codes.md": 1,+ "transformations.md": 2+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "err-2031",+ "query": "Why was my destination URL rejected with RW-2031?",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "error-codes.md",+ "network-security.md"+ ],+ "expected_text": "RW-2031",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 7.7312093,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 5.75021,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 4.6584964,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.269464,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 4.031956,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 3.6961586,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 3.5220945,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 8,+ "score": 3.4944685,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 9,+ "score": 3.3147504,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 3.3085356,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 11,+ "score": 3.0455709,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 3.013853,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 13,+ "score": 3.0125816,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 3.006339,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 15,+ "score": 2.862323,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.804049,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 2.4442608,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 18,+ "score": 2.4357595,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 2.4208562,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 20,+ "score": 2.021038,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "error-codes.md": 1,+ "network-security.md": 2+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "err-7001",+ "query": "What causes RW-7001?",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "error-codes.md",+ "regions.md"+ ],+ "expected_text": "RW-7001",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 4.55962,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 3.2900271,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 3,+ "score": 3.231194,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 3.0057564,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 2.8333864,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 6,+ "score": 2.6037674,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 2.4505215,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 8,+ "score": 2.2581377,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 9,+ "score": 2.2200274,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 2.1641731,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 2.1146564,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 2.0174422,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 1.6311694,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 14,+ "score": 1.4668834,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 15,+ "score": 1.456635,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 1.4466288,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 17,+ "score": 1.4390881,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 1.4314262,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 1.4135953,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "error-codes.md": 1,+ "regions.md": 3+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.9197207891481876,+ "ndcg@5": 0.9197207891481876,+ "ndcg@10": 0.9197207891481876,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "cfg-concurrency",+ "query": "Which setting limits how many requests hit my endpoint in parallel?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "configuration-reference.md"+ ],+ "expected_text": "delivery.max_concurrency",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "configuration-reference.md",+ "rank": 1,+ "score": 6.5323057,+ "snippet_found": true+ },+ {+ "filename": "rate-limits.md",+ "rank": 2,+ "score": 5.4767127,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 4.882698,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 4,+ "score": 4.4162087,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 4.319521,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 6,+ "score": 3.912937,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 7,+ "score": 3.9068096,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 8,+ "score": 3.5763764,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 9,+ "score": 3.4460037,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 10,+ "score": 3.442761,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 11,+ "score": 3.3418725,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 3.2560182,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 13,+ "score": 3.2276452,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 14,+ "score": 3.1950235,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 15,+ "score": 3.0526667,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 16,+ "score": 2.9480395,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 17,+ "score": 2.910755,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 2.5600786,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "configuration-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "cfg-dedupe-window",+ "query": "ingest.dedupe_window_seconds",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "configuration-reference.md"+ ],+ "expected_text": "ingest.dedupe_window_seconds",+ "chunks_returned": 8,+ "unique_files_returned": 1,+ "ranked_files": [+ {+ "filename": "configuration-reference.md",+ "rank": 1,+ "score": 3.756914,+ "snippet_found": true+ }+ ],+ "expected_ranks": {+ "configuration-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "cfg-explain",+ "query": "How do I find out which level a setting's value is coming from?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "configuration-reference.md",+ "cli-reference.md"+ ],+ "expected_text": "rwctl config explain",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "configuration-reference.md",+ "rank": 1,+ "score": 5.1910233,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 5.1194305,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 3,+ "score": 4.7757354,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 4,+ "score": 4.3601513,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 4.301612,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 6,+ "score": 4.265447,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 3.9866903,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 8,+ "score": 3.753189,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 3.693151,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 10,+ "score": 3.283983,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 3.2680726,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 12,+ "score": 3.261208,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 3.0610666,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 2.967939,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 15,+ "score": 2.8458738,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 16,+ "score": 2.8134208,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.7762527,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 18,+ "score": 2.6981857,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 2.6218233,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 20,+ "score": 2.5113726,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 21,+ "score": 2.4949827,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "configuration-reference.md": 1,+ "cli-reference.md": 8+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 0.5,+ "recall@5": 0.5,+ "recall@10": 0.5,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.6131471927654584,+ "ndcg@5": 0.6131471927654584,+ "ndcg@10": 0.6131471927654584,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "schema-validation",+ "query": "Can incoming payloads be checked against a schema and refused when they don't match?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "configuration-reference.md"+ ],+ "expected_text": "ingest.schema_validation",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "configuration-reference.md",+ "rank": 1,+ "score": 7.2857604,+ "snippet_found": true+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 2,+ "score": 4.1829777,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 3.9681184,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 3.9628394,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 3.8958888,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 6,+ "score": 3.8805716,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 3.7766762,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 8,+ "score": 3.7272916,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 3.6178882,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 10,+ "score": 3.5861323,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 11,+ "score": 3.5555618,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 12,+ "score": 3.5408092,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 13,+ "score": 3.2773092,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 3.2318847,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 15,+ "score": 3.1293461,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 16,+ "score": 3.0882106,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 17,+ "score": 2.9718976,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 18,+ "score": 2.8200388,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 19,+ "score": 2.7177334,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "configuration-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "filter-wildcard",+ "query": "How do I send every kind of invoice event, including nested types, to one endpoint?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "event-filtering.md"+ ],+ "expected_text": "invoice.**",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 6.599655,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 2,+ "score": 5.760765,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 3,+ "score": 5.569148,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 4,+ "score": 5.527859,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 5,+ "score": 5.472649,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 6,+ "score": 5.421536,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 5.261847,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 8,+ "score": 5.023038,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 9,+ "score": 4.937884,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 4.873948,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 4.553656,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 12,+ "score": 4.50159,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 13,+ "score": 4.4150305,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 4.278399,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 3.8844829,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 16,+ "score": 3.8552125,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 17,+ "score": 3.8130789,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 3.5139594,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 19,+ "score": 3.3147485,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "event-filtering.md": 7+ },+ "snippet_found": true,+ "passage_rank": 43,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.14285714285714285,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.0,+ "ndcg@10": 0.3333333333333333,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.023255813953488372+ }+ },+ {+ "id": "filter-missing-field",+ "query": "What happens when a filter refers to a field the event doesn't have?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "event-filtering.md"+ ],+ "expected_text": "it is not an error",+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "event-filtering.md",+ "rank": 1,+ "score": 6.3981233,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 6.132348,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 3,+ "score": 5.273807,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 4.9314632,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 4.871355,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 6,+ "score": 4.5469966,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 4.467554,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 4.247962,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 3.781559,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 10,+ "score": 3.587966,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 11,+ "score": 3.5403461,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 12,+ "score": 3.4803455,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 13,+ "score": 3.4259849,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 14,+ "score": 3.240863,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 3.217833,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 16,+ "score": 3.142176,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 17,+ "score": 3.012438,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "event-filtering.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "unrouted-events",+ "query": "What happens to an event that no route picks up?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "event-filtering.md"+ ],+ "expected_text": "unrouted",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "event-filtering.md",+ "rank": 1,+ "score": 6.7026176,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 6.576373,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 3,+ "score": 6.105928,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 5.184962,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 4.8381486,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 4.526141,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 4.250954,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 8,+ "score": 4.041794,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 4.0392456,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 3.9502084,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 3.8146112,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 3.8039114,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 13,+ "score": 3.4500139,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 14,+ "score": 3.4208,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 15,+ "score": 3.2320712,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 16,+ "score": 3.1246333,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 3.1182313,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 3.030007,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 19,+ "score": 2.9305377,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "event-filtering.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "route-fan-out",+ "query": "How many endpoints can one route deliver to?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "event-filtering.md"+ ],+ "expected_text": "up to 20 destinations",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 6.023015,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 2,+ "score": 5.745181,+ "snippet_found": true+ },+ {+ "filename": "configuration-reference.md",+ "rank": 3,+ "score": 5.1195517,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 4.844228,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 5,+ "score": 4.83197,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 6,+ "score": 4.238139,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 7,+ "score": 4.2260113,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 4.079329,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 3.8717012,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 10,+ "score": 3.8605428,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 11,+ "score": 3.7901826,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 12,+ "score": 3.5144677,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 13,+ "score": 3.5060587,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 14,+ "score": 3.367303,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 15,+ "score": 3.3218155,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 3.022322,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 17,+ "score": 2.9369514,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 18,+ "score": 2.922502,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 2.7017128,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "event-filtering.md": 2+ },+ "snippet_found": true,+ "passage_rank": 2,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.5,+ "ndcg@1": 0.0,+ "ndcg@3": 0.6309297535714575,+ "ndcg@5": 0.6309297535714575,+ "ndcg@10": 0.6309297535714575,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.5+ }+ },+ {+ "id": "strip-fields-before-delivery",+ "query": "How can I strip card numbers out of events before they are forwarded to a partner?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "transformations.md"+ ],+ "expected_text": "card_number",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 5.678779,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 2,+ "score": 4.841915,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 4.5332537,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 4.3789315,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 5,+ "score": 4.374852,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 6,+ "score": 4.303219,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 4.297907,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 4.2757826,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 9,+ "score": 4.0928054,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 10,+ "score": 4.013118,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 3.958882,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 12,+ "score": 3.8992069,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 13,+ "score": 3.830959,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 3.8095558,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 15,+ "score": 3.6439047,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 16,+ "score": 3.4828746,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 17,+ "score": 3.4731426,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 18,+ "score": 3.339641,+ "snippet_found": true+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 19,+ "score": 3.257053,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "transformations.md": 18+ },+ "snippet_found": true,+ "passage_rank": 44,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.0,+ "recall@10": 0.0,+ "reciprocal_rank": 0.05555555555555555,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.0,+ "ndcg@10": 0.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.022727272727272728+ }+ },+ {+ "id": "transform-network",+ "query": "Can transformation code call an external API?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "transformations.md"+ ],+ "expected_text": "No network access",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "error-codes.md",+ "rank": 1,+ "score": 4.3027563,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 2,+ "score": 3.7211428,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 3,+ "score": 3.5431662,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 3.4637728,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 5,+ "score": 3.4438903,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 6,+ "score": 3.3542087,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 7,+ "score": 3.1996362,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 3.1790874,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 9,+ "score": 3.1691957,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 10,+ "score": 2.7622945,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 11,+ "score": 2.7219043,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 12,+ "score": 2.7049937,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 2.6479259,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 14,+ "score": 2.5942392,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 15,+ "score": 2.5000453,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 2.412778,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 2.2223032,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 2.120767,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 2.0920844,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "transformations.md": 3+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "transform-rollback",+ "query": "I shipped a broken transformation, how do I go back to the previous one?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "transformations.md"+ ],+ "expected_text": "Make active",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.23054,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 2,+ "score": 4.2707195,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 3,+ "score": 4.0321445,+ "snippet_found": true+ },+ {+ "filename": "rate-limits.md",+ "rank": 4,+ "score": 3.5525649,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 3.4241276,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 6,+ "score": 3.4105673,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 3.4089398,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.4029415,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 3.3298483,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 3.2877781,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 11,+ "score": 3.2613475,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 3.2458348,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 3.2236872,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 14,+ "score": 3.2179403,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 15,+ "score": 3.1666298,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 16,+ "score": 2.8349867,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 2.7176538,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 18,+ "score": 2.5744977,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 2.480778,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "transformations.md": 3+ },+ "snippet_found": true,+ "passage_rank": 23,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.043478260869565216+ }+ },+ {+ "id": "plan-scale-price",+ "query": "What does the Scale plan cost and how many events come with it?",+ "tags": [+ "keyword"+ ],+ "expected_files": [+ "billing-plans.md"+ ],+ "expected_text": "20 million",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "data-retention.md",+ "rank": 1,+ "score": 6.165932,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 2,+ "score": 5.6396055,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 3,+ "score": 5.595186,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.96609,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 4.8011065,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 4.7937565,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 4.7090893,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 4.6951523,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 9,+ "score": 4.6432734,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 10,+ "score": 4.542534,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 11,+ "score": 3.9930236,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 3.9314911,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 3.7961717,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 14,+ "score": 3.7279844,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 3.7183979,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 16,+ "score": 3.5648823,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 17,+ "score": 3.5081198,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 18,+ "score": 3.369672,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "billing-plans.md": 3+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "over-quota",+ "query": "What happens if we send more events in a month than our plan includes?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "billing-plans.md"+ ],+ "expected_text": "billed as overage",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "billing-plans.md",+ "rank": 1,+ "score": 6.611384,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 5.573196,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 5.069791,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 4,+ "score": 4.896354,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 5,+ "score": 4.8308477,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 6,+ "score": 4.6927853,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 7,+ "score": 4.615057,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 8,+ "score": 4.568859,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 9,+ "score": 4.510277,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 4.4583926,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 4.3350053,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 4.3333607,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 13,+ "score": 4.2085896,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 3.864415,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 15,+ "score": 3.6676457,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 16,+ "score": 3.6663413,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 17,+ "score": 3.626868,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 18,+ "score": 3.5489278,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "billing-plans.md": 1+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "spending-cap",+ "query": "How do I make sure we never get a surprise bill?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "billing-plans.md"+ ],+ "expected_text": "hard cap",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "cli-reference.md",+ "rank": 1,+ "score": 4.176035,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 2,+ "score": 3.7238424,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 3,+ "score": 3.373507,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 3.3062384,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 3.1803827,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 2.8217483,+ "snippet_found": true+ },+ {+ "filename": "signing-secrets.md",+ "rank": 7,+ "score": 2.8194714,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 2.7950668,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 9,+ "score": 2.7918391,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 10,+ "score": 2.7697334,+ "snippet_found": true+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 11,+ "score": 2.3493757,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 12,+ "score": 2.2208354,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 13,+ "score": 2.1967921,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 14,+ "score": 2.0774598,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 15,+ "score": 2.0550961,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 16,+ "score": 2.0002537,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 17,+ "score": 1.9782275,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 18,+ "score": 1.8935627,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 19,+ "score": 1.8432963,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "billing-plans.md": 10+ },+ "snippet_found": true,+ "passage_rank": 11,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.1,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.0,+ "ndcg@10": 0.2890648263178879,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.09090909090909091+ }+ },+ {+ "id": "downgrade-timing",+ "query": "When does a plan downgrade kick in?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "billing-plans.md"+ ],+ "expected_text": "end of the current billing period",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "billing-plans.md",+ "rank": 1,+ "score": 4.5011973,+ "snippet_found": true+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 2,+ "score": 3.9330368,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 3.6819746,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 4,+ "score": 3.3807042,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 5,+ "score": 3.145202,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 6,+ "score": 2.9044037,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 2.6842203,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 8,+ "score": 2.652769,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 9,+ "score": 2.559432,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 10,+ "score": 2.5516438,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 2.5177975,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 12,+ "score": 2.4889445,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 2.4745228,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 14,+ "score": 2.397379,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 2.3737328,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.3688858,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 17,+ "score": 2.2727196,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 18,+ "score": 2.255967,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 19,+ "score": 2.1137109,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "billing-plans.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "nonprofit-discount",+ "query": "Is there a discount for nonprofits?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "billing-plans.md"+ ],+ "expected_text": "50 percent off",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 3.192665,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 2,+ "score": 3.1329155,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 2.7976594,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 4,+ "score": 2.500914,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 2.431467,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 6,+ "score": 2.3387825,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 7,+ "score": 2.1462817,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 2.138791,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 2.0935802,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 1.9763438,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 11,+ "score": 1.9572263,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 12,+ "score": 1.9540287,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 1.8847063,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 14,+ "score": 1.8813934,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 1.8805516,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 16,+ "score": 1.8092102,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 17,+ "score": 1.799421,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 1.7747855,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 1.7663033,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "billing-plans.md": 2+ },+ "snippet_found": true,+ "passage_rank": 2,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.5,+ "ndcg@1": 0.0,+ "ndcg@3": 0.6309297535714575,+ "ndcg@5": 0.6309297535714575,+ "ndcg@10": 0.6309297535714575,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.5+ }+ },+ {+ "id": "card-declined",+ "query": "What happens if our card is declined?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "invoices-and-payments.md"+ ],+ "expected_text": "suspended",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 4.5424986,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 3.1390238,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 3,+ "score": 2.9242349,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 2.853714,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 2.7698298,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 6,+ "score": 2.6912606,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 2.6068764,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 8,+ "score": 2.4017735,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 9,+ "score": 2.3534894,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 10,+ "score": 2.3194094,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 2.2925973,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 2.2087455,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 2.170509,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 14,+ "score": 2.1145196,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 2.0395908,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 16,+ "score": 1.8723415,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 17,+ "score": 1.8695025,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 18,+ "score": 1.8511734,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "invoices-and-payments.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "invoice-download",+ "query": "Where can I download past invoices?",+ "tags": [+ "keyword"+ ],+ "expected_files": [+ "invoices-and-payments.md"+ ],+ "expected_text": "PDF",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 5.479479,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 3.3870986,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 3,+ "score": 3.3651319,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 4,+ "score": 3.1572304,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 2.869647,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 2.8024464,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 7,+ "score": 2.6480045,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 2.558901,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 9,+ "score": 2.514241,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 10,+ "score": 2.4659653,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 2.4333763,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 12,+ "score": 2.4234314,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 13,+ "score": 2.3397257,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 14,+ "score": 2.3184114,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.2858236,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 2.261516,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 17,+ "score": 2.2397711,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 18,+ "score": 1.9229008,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 19,+ "score": 1.9125963,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 20,+ "score": 1.8740361,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "invoices-and-payments.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "annual-refund",+ "query": "Can I get my money back on an annual subscription?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "invoices-and-payments.md"+ ],+ "expected_text": "30 days of purchase",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 4.030309,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 2,+ "score": 3.4916003,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 3,+ "score": 3.1594803,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 3.083531,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 2.862523,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 2.7023883,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 7,+ "score": 2.6665974,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 8,+ "score": 2.5352874,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 9,+ "score": 2.4236565,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.3765519,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 11,+ "score": 2.3002856,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 12,+ "score": 2.2416017,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 2.06355,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 14,+ "score": 1.9668009,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 15,+ "score": 1.899621,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 1.8948866,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 17,+ "score": 1.8911364,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 1.875198,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 19,+ "score": 1.7308294,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "invoices-and-payments.md": 3+ },+ "snippet_found": true,+ "passage_rank": 39,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.02564102564102564+ }+ },+ {+ "id": "pay-by-wire",+ "query": "Can we pay by wire instead of by card?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "invoices-and-payments.md"+ ],+ "expected_text": "bank transfer",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 5.018153,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 3.2159462,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 3.201638,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 4,+ "score": 3.147363,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 5,+ "score": 2.6815596,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 6,+ "score": 2.6559532,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 7,+ "score": 2.5634036,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 8,+ "score": 2.364353,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 9,+ "score": 2.3472905,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 10,+ "score": 2.2389264,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 2.2166698,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 12,+ "score": 2.180029,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 2.1364298,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 14,+ "score": 2.1318357,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 15,+ "score": 2.0653362,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 16,+ "score": 1.992836,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 17,+ "score": 1.948524,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 18,+ "score": 1.9307742,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 19,+ "score": 1.7839562,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "invoices-and-payments.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "payload-retention-scale",+ "query": "How long are event bodies stored on the Scale plan?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "data-retention.md"+ ],+ "expected_text": "These are maximums",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "data-retention.md",+ "rank": 1,+ "score": 7.134757,+ "snippet_found": true+ },+ {+ "filename": "billing-plans.md",+ "rank": 2,+ "score": 5.646868,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 3,+ "score": 5.632083,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 4,+ "score": 5.5159082,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 5,+ "score": 5.322894,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 6,+ "score": 4.917618,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 4.646406,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 8,+ "score": 4.593455,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 9,+ "score": 4.1423883,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 10,+ "score": 4.102504,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 11,+ "score": 3.5780063,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 3.4474115,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 3.1836896,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 3.1513236,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 15,+ "score": 3.037868,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 16,+ "score": 2.9461775,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.7285311,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 18,+ "score": 2.6023915,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "data-retention.md": 1+ },+ "snippet_found": true,+ "passage_rank": 14,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.07142857142857142+ }+ },+ {+ "id": "erasure-request",+ "query": "A customer asked us to erase their personal data, how do I remove their events?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "data-retention.md"+ ],+ "expected_text": "Delete events",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "data-retention.md",+ "rank": 1,+ "score": 7.6901236,+ "snippet_found": true+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 2,+ "score": 6.390702,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 3,+ "score": 5.357597,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 4.2579355,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 5,+ "score": 4.2515087,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 6,+ "score": 4.1859593,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 4.102975,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 8,+ "score": 3.948564,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 9,+ "score": 3.9145777,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 10,+ "score": 3.8292284,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 3.7746987,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 12,+ "score": 3.7034235,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 13,+ "score": 3.6457257,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 3.4510388,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 3.381026,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 16,+ "score": 3.1673608,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 17,+ "score": 3.1267197,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 18,+ "score": 2.711932,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "data-retention.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "never-store-field",+ "query": "How do I keep a sensitive field from ever being saved?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "data-retention.md",+ "configuration-reference.md"+ ],+ "expected_text": "retention.redact_fields",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "transformations.md",+ "rank": 1,+ "score": 4.269417,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 2,+ "score": 4.132926,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 4.115541,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 4,+ "score": 3.9203172,+ "snippet_found": true+ },+ {+ "filename": "single-sign-on.md",+ "rank": 5,+ "score": 3.7807415,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 6,+ "score": 3.7652113,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 3.6132348,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.4858923,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 9,+ "score": 3.1855412,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 10,+ "score": 3.0955997,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 11,+ "score": 3.075557,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 12,+ "score": 2.8341694,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 13,+ "score": 2.6905017,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 14,+ "score": 2.600214,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.5657752,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 2.553431,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 17,+ "score": 2.3961258,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 18,+ "score": 2.3078952,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 2.2757673,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 20,+ "score": 2.2442474,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "data-retention.md": 4,+ "configuration-reference.md": 17+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.5,+ "recall@10": 0.5,+ "reciprocal_rank": 0.25,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.2640681225725909,+ "ndcg@10": 0.2640681225725909,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "export-to-own-storage",+ "query": "Can I keep events for longer than the plan allows by copying them to storage we own?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "data-retention.md"+ ],+ "expected_text": "scheduled export",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "data-retention.md",+ "rank": 1,+ "score": 7.004419,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 5.72364,+ "snippet_found": true+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 5.051481,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 4,+ "score": 4.8311186,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 4.7930336,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 6,+ "score": 4.558029,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 7,+ "score": 4.422286,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 8,+ "score": 4.2385845,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 4.049222,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 10,+ "score": 4.0140843,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 3.8202207,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 12,+ "score": 3.7686496,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 13,+ "score": 3.7210777,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 14,+ "score": 3.7047937,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 3.7034209,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 16,+ "score": 3.6294847,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 17,+ "score": 3.6183372,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 3.476426,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 19,+ "score": 3.2785215,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 20,+ "score": 3.2107797,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "data-retention.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "replay-event-gone",+ "query": "Replay fails saying the event isn't available any more, why?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "data-retention.md",+ "troubleshooting.md"+ ],+ "expected_text": null,+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.111964,+ "snippet_found": null+ },+ {+ "filename": "delivery-retries.md",+ "rank": 2,+ "score": 4.6264553,+ "snippet_found": null+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 4.382534,+ "snippet_found": null+ },+ {+ "filename": "error-codes.md",+ "rank": 4,+ "score": 4.204933,+ "snippet_found": null+ },+ {+ "filename": "event-filtering.md",+ "rank": 5,+ "score": 4.008878,+ "snippet_found": null+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 4.000078,+ "snippet_found": null+ },+ {+ "filename": "signing-secrets.md",+ "rank": 7,+ "score": 3.8352492,+ "snippet_found": null+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 3.6457763,+ "snippet_found": null+ },+ {+ "filename": "data-retention.md",+ "rank": 9,+ "score": 3.5607314,+ "snippet_found": null+ },+ {+ "filename": "billing-plans.md",+ "rank": 10,+ "score": 3.5086422,+ "snippet_found": null+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 11,+ "score": 3.5036902,+ "snippet_found": null+ },+ {+ "filename": "configuration-reference.md",+ "rank": 12,+ "score": 3.4338822,+ "snippet_found": null+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 3.0842357,+ "snippet_found": null+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 3.065458,+ "snippet_found": null+ },+ {+ "filename": "network-security.md",+ "rank": 15,+ "score": 2.9612713,+ "snippet_found": null+ },+ {+ "filename": "getting-started.md",+ "rank": 16,+ "score": 2.9603195,+ "snippet_found": null+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 17,+ "score": 2.8865094,+ "snippet_found": null+ },+ {+ "filename": "api-keys.md",+ "rank": 18,+ "score": 2.8189383,+ "snippet_found": null+ }+ ],+ "expected_ranks": {+ "data-retention.md": 9,+ "troubleshooting.md": 1+ },+ "snippet_found": null,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 0.5,+ "recall@5": 0.5,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 0.6131471927654584,+ "ndcg@5": 0.6131471927654584,+ "ndcg@10": 0.7977228895450266+ }+ },+ {+ "id": "eu-residency",+ "query": "We need our data to stay inside the EU, is that possible?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "regions.md"+ ],+ "expected_text": "eu-central",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "regions.md",+ "rank": 1,+ "score": 5.606211,+ "snippet_found": true+ },+ {+ "filename": "api-keys.md",+ "rank": 2,+ "score": 3.7901576,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 3,+ "score": 3.3422961,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 4,+ "score": 3.0132427,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 5,+ "score": 2.9385717,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 6,+ "score": 2.8419957,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 7,+ "score": 2.7693527,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 8,+ "score": 2.7328167,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 9,+ "score": 2.4419742,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 2.4058669,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 11,+ "score": 2.3926773,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 12,+ "score": 2.3721557,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 13,+ "score": 2.3270187,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 2.2621386,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 2.2538724,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 16,+ "score": 2.1800587,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 17,+ "score": 2.1209936,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 18,+ "score": 2.111018,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 19,+ "score": 1.9036365,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "regions.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "move-region",+ "query": "Can I move an existing workspace to another region?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "regions.md"+ ],+ "expected_text": "cannot be changed",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "regions.md",+ "rank": 1,+ "score": 7.4162574,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.9218082,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 4.279954,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 4,+ "score": 3.787121,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 5,+ "score": 3.5782912,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 6,+ "score": 3.4162853,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 3.3293962,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 8,+ "score": 3.0762732,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 3.0027943,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.9621854,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 11,+ "score": 2.8567605,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 2.7653418,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 13,+ "score": 2.742486,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 14,+ "score": 2.7398138,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.7382836,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 16,+ "score": 2.6832218,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 17,+ "score": 2.652599,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 18,+ "score": 2.6192703,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 19,+ "score": 2.2097597,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 20,+ "score": 2.1313481,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "regions.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "firewall-ips",+ "query": "Which IP addresses do I need to let through our firewall so deliveries arrive?",+ "tags": [+ "semantic",+ "multi-doc"+ ],+ "expected_files": [+ "regions.md",+ "network-security.md"+ ],+ "expected_text": "rwctl network egress-ips",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "regions.md",+ "rank": 1,+ "score": 7.6239576,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 7.329286,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 4.260872,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 4,+ "score": 3.736021,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 5,+ "score": 3.5849426,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 6,+ "score": 3.4546218,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 7,+ "score": 3.4361572,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 8,+ "score": 3.4263103,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 9,+ "score": 3.3875096,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 10,+ "score": 3.3755388,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 11,+ "score": 3.2378645,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 12,+ "score": 3.1179535,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 13,+ "score": 3.0980186,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 14,+ "score": 3.062451,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 15,+ "score": 3.0315938,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 16,+ "score": 2.801557,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 2.7423558,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 18,+ "score": 2.728594,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 19,+ "score": 2.5501351,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 20,+ "score": 2.5037901,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 21,+ "score": 2.3320217,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "regions.md": 1,+ "network-security.md": 2+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "alert-on-failing-endpoint",+ "query": "How do I get notified when an endpoint starts failing?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "monitoring-and-alerts.md"+ ],+ "expected_text": "delivery.success_rate",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 1,+ "score": 4.7551217,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 2,+ "score": 4.4837484,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 3.97679,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 4,+ "score": 3.7504718,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 3.7289722,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 6,+ "score": 3.680981,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 3.4579203,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 3.360943,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 9,+ "score": 3.2502306,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 10,+ "score": 3.0607262,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 11,+ "score": 3.031942,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 12,+ "score": 2.939257,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 13,+ "score": 2.9016738,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 14,+ "score": 2.7974367,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 15,+ "score": 2.660655,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.4786308,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.3564715,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 2.2467048,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "monitoring-and-alerts.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "metric-dlq-depth",+ "query": "dlq.depth",+ "tags": [+ "exact-token",+ "multi-doc"+ ],+ "expected_files": [+ "monitoring-and-alerts.md",+ "dead-letter-queue.md"+ ],+ "expected_text": "dlq.depth",+ "chunks_returned": 8,+ "unique_files_returned": 2,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 3.1213505,+ "snippet_found": true+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 2,+ "score": 2.994694,+ "snippet_found": true+ }+ ],+ "expected_ranks": {+ "monitoring-and-alerts.md": 2,+ "dead-letter-queue.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "alert-noise",+ "query": "How do I stop an alert from notifying us over and over?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "monitoring-and-alerts.md"+ ],+ "expected_text": "cooldown",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 1,+ "score": 5.3605433,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 2,+ "score": 4.4922824,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 3,+ "score": 3.8867335,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 4,+ "score": 3.680799,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 3.500159,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 3.3970802,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 7,+ "score": 3.2065027,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 8,+ "score": 3.1697018,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 3.005413,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 10,+ "score": 2.8459678,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 11,+ "score": 2.7075675,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 12,+ "score": 2.6995163,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 2.6899762,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 2.6840582,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 15,+ "score": 2.664556,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.4973686,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.4592614,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 18,+ "score": 2.4189043,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 19,+ "score": 2.3941422,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 20,+ "score": 2.3926606,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 21,+ "score": 2.2645342,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "monitoring-and-alerts.md": 1+ },+ "snippet_found": true,+ "passage_rank": 17,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.058823529411764705+ }+ },+ {+ "id": "metrics-history",+ "query": "How far back do the charts go?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "monitoring-and-alerts.md"+ ],+ "expected_text": "13 months",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 1,+ "score": 3.7359376,+ "snippet_found": true+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 2,+ "score": 3.1819804,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 3,+ "score": 2.9821012,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 4,+ "score": 2.8422427,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 5,+ "score": 2.8005157,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 6,+ "score": 2.783891,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 2.782668,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 8,+ "score": 2.712003,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 9,+ "score": 2.498731,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 10,+ "score": 2.2379708,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 11,+ "score": 2.218436,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 12,+ "score": 2.206134,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 2.102532,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 2.0636818,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 15,+ "score": 2.0426698,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 16,+ "score": 2.0245354,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 17,+ "score": 1.9765356,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 1.8800985,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 19,+ "score": 1.7948415,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 20,+ "score": 1.7783153,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "monitoring-and-alerts.md": 1+ },+ "snippet_found": true,+ "passage_rank": 14,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.07142857142857142+ }+ },+ {+ "id": "audit-who-changed",+ "query": "How can I find out who changed a destination's URL?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "audit-log.md"+ ],+ "expected_text": "destination.url_changed",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "invoices-and-payments.md",+ "rank": 1,+ "score": 4.8638945,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 4.4864283,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 3,+ "score": 4.4840837,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 4.211418,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 5,+ "score": 4.071476,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 6,+ "score": 4.007015,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 3.974052,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 3.8140574,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 9,+ "score": 3.7421665,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 10,+ "score": 3.5506918,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 11,+ "score": 3.4370897,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 12,+ "score": 3.4214826,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 13,+ "score": 3.28876,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 3.161325,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 3.1208224,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 16,+ "score": 2.7832265,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 2.6834779,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 18,+ "score": 2.5932093,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 19,+ "score": 2.4568262,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "audit-log.md": 3+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "audit-retention",+ "query": "How long is the record of admin actions kept?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "audit-log.md"+ ],+ "expected_text": "7 years",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "audit-log.md",+ "rank": 1,+ "score": 6.7694893,+ "snippet_found": true+ },+ {+ "filename": "data-retention.md",+ "rank": 2,+ "score": 5.902327,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 3,+ "score": 4.3268433,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 4,+ "score": 4.1397276,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 4.104513,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 6,+ "score": 4.062165,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 4.053916,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 3.2497015,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 3.210817,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 10,+ "score": 3.150314,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 11,+ "score": 3.0505908,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 12,+ "score": 2.9865234,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 13,+ "score": 2.8661394,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 2.650631,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.542187,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 16,+ "score": 2.2279477,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 2.0776277,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 18,+ "score": 1.9082611,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 19,+ "score": 1.9064578,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "audit-log.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "audit-tamper",+ "query": "Could an admin cover their tracks by deleting log entries?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "audit-log.md"+ ],+ "expected_text": "hash",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "roles-and-permissions.md",+ "rank": 1,+ "score": 5.3748984,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 2,+ "score": 5.285913,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 3,+ "score": 5.285846,+ "snippet_found": true+ },+ {+ "filename": "delivery-retries.md",+ "rank": 4,+ "score": 4.522437,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.9469934,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 6,+ "score": 3.3514473,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 3.1368482,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 8,+ "score": 3.126635,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 9,+ "score": 2.75564,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 10,+ "score": 2.7482202,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 11,+ "score": 2.715976,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 12,+ "score": 2.5803761,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 13,+ "score": 2.460242,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 14,+ "score": 2.3662038,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.3606703,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 2.197319,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 17,+ "score": 2.1412895,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 18,+ "score": 2.0583878,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "audit-log.md": 3+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "support-access",+ "query": "Can Relaywise staff look at our payloads?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "audit-log.md"+ ],+ "expected_text": "Support access",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "audit-log.md",+ "rank": 1,+ "score": 4.0565033,+ "snippet_found": true+ },+ {+ "filename": "transformations.md",+ "rank": 2,+ "score": 3.8535635,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 3,+ "score": 3.7648094,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 4,+ "score": 3.761508,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.7174695,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 6,+ "score": 3.6284103,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 7,+ "score": 3.3209238,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 3.0343482,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 9,+ "score": 2.9823442,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 10,+ "score": 2.81755,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 2.5708063,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 2.5682056,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 13,+ "score": 2.5657067,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 14,+ "score": 2.5156288,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 15,+ "score": 2.5056129,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 16,+ "score": 2.4120002,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 17,+ "score": 2.404485,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 18,+ "score": 2.3355746,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 19,+ "score": 2.2381802,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 20,+ "score": 2.1149168,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "audit-log.md": 1+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "local-development",+ "query": "How do I receive webhooks on my laptop while developing?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "cli-reference.md"+ ],+ "expected_text": "rwctl listen",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "cli-reference.md",+ "rank": 1,+ "score": 5.920323,+ "snippet_found": true+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 2,+ "score": 3.614477,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 3,+ "score": 3.504744,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 4,+ "score": 3.2636237,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 5,+ "score": 3.0580826,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 6,+ "score": 3.0453985,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 7,+ "score": 2.8457193,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 8,+ "score": 2.6562397,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 9,+ "score": 2.5232592,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 10,+ "score": 2.3091094,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 2.298862,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 12,+ "score": 2.2715812,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 13,+ "score": 2.1830935,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 14,+ "score": 1.984105,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 15,+ "score": 1.9079537,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 16,+ "score": 1.9055259,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 1.8890364,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 1.8846616,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 1.8736366,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "cli-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "cli-in-pipeline",+ "query": "How do I authenticate the command-line tool in a CI pipeline?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "cli-reference.md"+ ],+ "expected_text": "RELAYWISE_API_KEY",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "cli-reference.md",+ "rank": 1,+ "score": 6.744873,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 6.4332047,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 3,+ "score": 4.639228,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 4.624231,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 4.0451546,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 6,+ "score": 3.0904324,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 2.9595504,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 2.9179506,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 2.8892794,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 10,+ "score": 2.8273864,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 11,+ "score": 2.6481714,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 12,+ "score": 2.6116657,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 13,+ "score": 2.4750228,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 14,+ "score": 2.3840961,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.3526254,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 2.3453667,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 17,+ "score": 2.329196,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 2.3105922,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 19,+ "score": 2.0088358,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "cli-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 6,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.16666666666666666+ }+ },+ {+ "id": "cli-exit-code-5",+ "query": "What does exit code 5 from rwctl mean?",+ "tags": [+ "exact-token"+ ],+ "expected_files": [+ "cli-reference.md"+ ],+ "expected_text": "Rate limited",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "cli-reference.md",+ "rank": 1,+ "score": 6.5858765,+ "snippet_found": true+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 2,+ "score": 4.121351,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 3,+ "score": 3.7905457,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 4,+ "score": 3.750536,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 5,+ "score": 3.116803,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 6,+ "score": 3.0487542,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 2.8950555,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 8,+ "score": 2.7257786,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 9,+ "score": 2.7019591,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 10,+ "score": 2.624712,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 2.6233444,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 12,+ "score": 2.5043364,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 13,+ "score": 2.4628139,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 14,+ "score": 2.430686,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.3871965,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 2.359127,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 2.3341298,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 18,+ "score": 2.2257278,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 19,+ "score": 2.1425583,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 20,+ "score": 2.125866,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 21,+ "score": 2.0921934,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "cli-reference.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "client-certificates",+ "query": "Our endpoint requires callers to present a client certificate, is that supported?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "network-security.md"+ ],+ "expected_text": "mutual TLS",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "network-security.md",+ "rank": 1,+ "score": 9.485439,+ "snippet_found": true+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.8805037,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 3,+ "score": 4.1398563,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 4,+ "score": 3.9929626,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 5,+ "score": 3.6816976,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 6,+ "score": 3.5573359,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 7,+ "score": 3.1600873,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 3.1363287,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 9,+ "score": 3.1233253,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 3.0414858,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 11,+ "score": 2.9609323,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 2.8957937,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 2.7488675,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 14,+ "score": 2.703147,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 15,+ "score": 2.6985838,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 16,+ "score": 2.6221676,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 17,+ "score": 2.2819054,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 18,+ "score": 2.1841009,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "network-security.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "restrict-senders",+ "query": "Can I restrict which servers are allowed to send events to us?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "network-security.md"+ ],+ "expected_text": "ingest allowlist",+ "chunks_returned": 50,+ "unique_files_returned": 17,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 4.716089,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.5632486,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 4.4465055,+ "snippet_found": true+ },+ {+ "filename": "rate-limits.md",+ "rank": 4,+ "score": 4.4074636,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 5,+ "score": 4.3561697,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 6,+ "score": 4.2924013,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 7,+ "score": 4.2620754,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 8,+ "score": 4.0919485,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 9,+ "score": 4.0235558,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 10,+ "score": 3.9080744,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 3.8798714,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 3.6969318,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 13,+ "score": 3.6393707,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 14,+ "score": 3.5710607,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 15,+ "score": 3.4468415,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 3.0487945,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 17,+ "score": 2.857802,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "network-security.md": 3+ },+ "snippet_found": true,+ "passage_rank": 3,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.3333333333333333+ }+ },+ {+ "id": "enforce-two-step",+ "query": "How do I force everyone to use two-step login?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "network-security.md"+ ],+ "expected_text": "Require two-factor",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "single-sign-on.md",+ "rank": 1,+ "score": 4.196766,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.1229253,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 3.2927356,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 4,+ "score": 3.2802827,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 5,+ "score": 3.2128882,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 6,+ "score": 3.0439441,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 7,+ "score": 2.7201033,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 8,+ "score": 2.644678,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 9,+ "score": 2.5467582,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 10,+ "score": 2.535389,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 11,+ "score": 2.4999475,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 12,+ "score": 2.4157667,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 13,+ "score": 2.4001741,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 14,+ "score": 2.3948617,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 15,+ "score": 2.3878696,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 16,+ "score": 2.282506,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.2025251,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 2.0695238,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 19,+ "score": 1.9594917,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 20,+ "score": 1.9291121,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 21,+ "score": 1.796094,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "network-security.md": 3+ },+ "snippet_found": true,+ "passage_rank": 9,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.3333333333333333,+ "ndcg@1": 0.0,+ "ndcg@3": 0.5,+ "ndcg@5": 0.5,+ "ndcg@10": 0.5,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.1111111111111111+ }+ },+ {+ "id": "encryption-at-rest",+ "query": "Is stored data encrypted?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "network-security.md"+ ],+ "expected_text": "AES-256",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "network-security.md",+ "rank": 1,+ "score": 4.051931,+ "snippet_found": true+ },+ {+ "filename": "data-retention.md",+ "rank": 2,+ "score": 3.8924236,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 3,+ "score": 3.4245567,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 4,+ "score": 2.852664,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 2.5582569,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 6,+ "score": 2.4678826,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 7,+ "score": 2.4120307,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 8,+ "score": 2.2670894,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 9,+ "score": 2.194287,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 10,+ "score": 2.1178303,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 11,+ "score": 1.8928216,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 12,+ "score": 1.864538,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 13,+ "score": 1.8635015,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 14,+ "score": 1.7791486,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 15,+ "score": 1.5821353,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 16,+ "score": 1.5647495,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 17,+ "score": 1.5531394,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 1.5409894,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 19,+ "score": 1.4597204,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "network-security.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "nothing-arriving",+ "query": "My endpoint isn't getting any events, where do I start looking?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "troubleshooting.md"+ ],+ "expected_text": "Was the event accepted",+ "chunks_returned": 50,+ "unique_files_returned": 19,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.6265793,+ "snippet_found": true+ },+ {+ "filename": "getting-started.md",+ "rank": 2,+ "score": 4.9406214,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 3,+ "score": 4.242301,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 4,+ "score": 4.122987,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 5,+ "score": 4.0675335,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 6,+ "score": 3.9629734,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 7,+ "score": 3.9350436,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 8,+ "score": 3.9290373,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 9,+ "score": 3.8895578,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 10,+ "score": 3.573028,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 11,+ "score": 3.5166721,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 12,+ "score": 3.334948,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 13,+ "score": 3.2872982,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 3.243579,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 15,+ "score": 3.0642722,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 16,+ "score": 3.0577633,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 17,+ "score": 3.028301,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 2.8070688,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 19,+ "score": 2.774746,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "troubleshooting.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "lost-authenticator",+ "query": "I lost my authenticator app, how do I get back in?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "troubleshooting.md"+ ],+ "expected_text": "second factor",+ "chunks_returned": 50,+ "unique_files_returned": 21,+ "ranked_files": [+ {+ "filename": "troubleshooting.md",+ "rank": 1,+ "score": 5.154069,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 2,+ "score": 4.039789,+ "snippet_found": true+ },+ {+ "filename": "single-sign-on.md",+ "rank": 3,+ "score": 3.4582567,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 4,+ "score": 3.15412,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 5,+ "score": 3.0289927,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 3.0060077,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 7,+ "score": 2.9968257,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 8,+ "score": 2.9520168,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 9,+ "score": 2.8144894,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 10,+ "score": 2.6343243,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 11,+ "score": 2.5093923,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 12,+ "score": 2.4954455,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 2.4666529,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 14,+ "score": 2.3090854,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 15,+ "score": 2.2876267,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 16,+ "score": 2.262417,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 17,+ "score": 2.1686492,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 18,+ "score": 2.117509,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 19,+ "score": 2.0579886,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 20,+ "score": 1.9386961,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 21,+ "score": 1.9223454,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "troubleshooting.md": 1+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 1.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ },+ {+ "id": "first-test-event",+ "query": "What is the quickest way to send a first test event?",+ "tags": [+ "semantic"+ ],+ "expected_files": [+ "getting-started.md"+ ],+ "expected_text": "202 Accepted",+ "chunks_returned": 50,+ "unique_files_returned": 18,+ "ranked_files": [+ {+ "filename": "dead-letter-queue.md",+ "rank": 1,+ "score": 5.160555,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 2,+ "score": 4.864584,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 3,+ "score": 4.5930367,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 4,+ "score": 4.4245358,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 5,+ "score": 4.3252854,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 6,+ "score": 4.2423105,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 7,+ "score": 4.239122,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 8,+ "score": 4.103107,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 9,+ "score": 3.8884325,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 10,+ "score": 3.7158773,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 11,+ "score": 3.655969,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 12,+ "score": 3.6182408,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 13,+ "score": 3.407936,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 14,+ "score": 3.300753,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 15,+ "score": 3.2260537,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 16,+ "score": 3.218966,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 3.0701206,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 18,+ "score": 3.029202,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "getting-started.md": 7+ },+ "snippet_found": false,+ "passage_rank": null,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 0.0,+ "recall@5": 0.0,+ "recall@10": 0.0,+ "reciprocal_rank": 0.0,+ "ndcg@1": 0.0,+ "ndcg@3": 0.0,+ "ndcg@5": 0.0,+ "ndcg@10": 0.0,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 0.0,+ "passage_hit@10": 0.0,+ "passage_rr": 0.0+ }+ },+ {+ "id": "source-vs-destination",+ "query": "What is the difference between a source and a destination?",+ "tags": [+ "keyword"+ ],+ "expected_files": [+ "getting-started.md"+ ],+ "expected_text": "Where events come from",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "cli-reference.md",+ "rank": 1,+ "score": 4.9485564,+ "snippet_found": false+ },+ {+ "filename": "getting-started.md",+ "rank": 2,+ "score": 4.9001284,+ "snippet_found": true+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 3,+ "score": 3.985413,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 4,+ "score": 3.8785093,+ "snippet_found": false+ },+ {+ "filename": "network-security.md",+ "rank": 5,+ "score": 3.8600652,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 6,+ "score": 3.7427316,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 7,+ "score": 3.673921,+ "snippet_found": false+ },+ {+ "filename": "error-codes.md",+ "rank": 8,+ "score": 3.6201513,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 9,+ "score": 3.4754732,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 10,+ "score": 3.3986077,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 11,+ "score": 3.2834342,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 12,+ "score": 3.2481108,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 13,+ "score": 3.2231283,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 14,+ "score": 3.2037265,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 15,+ "score": 3.149214,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 16,+ "score": 3.0159397,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 17,+ "score": 2.9234078,+ "snippet_found": false+ },+ {+ "filename": "data-retention.md",+ "rank": 18,+ "score": 2.8746972,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 19,+ "score": 2.7781506,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 20,+ "score": 2.455618,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "getting-started.md": 2+ },+ "snippet_found": true,+ "passage_rank": 4,+ "metrics": {+ "recall@1": 0.0,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 0.5,+ "ndcg@1": 0.0,+ "ndcg@3": 0.6309297535714575,+ "ndcg@5": 0.6309297535714575,+ "ndcg@10": 0.6309297535714575,+ "passage_hit@1": 0.0,+ "passage_hit@3": 0.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 0.25+ }+ },+ {+ "id": "plain-http-destination",+ "query": "Can a destination use a plain http URL?",+ "tags": [+ "keyword",+ "multi-doc"+ ],+ "expected_files": [+ "getting-started.md",+ "error-codes.md"+ ],+ "expected_text": "localhost",+ "chunks_returned": 50,+ "unique_files_returned": 20,+ "ranked_files": [+ {+ "filename": "getting-started.md",+ "rank": 1,+ "score": 6.999161,+ "snippet_found": true+ },+ {+ "filename": "error-codes.md",+ "rank": 2,+ "score": 6.419933,+ "snippet_found": true+ },+ {+ "filename": "network-security.md",+ "rank": 3,+ "score": 3.9198084,+ "snippet_found": false+ },+ {+ "filename": "cli-reference.md",+ "rank": 4,+ "score": 3.8565774,+ "snippet_found": false+ },+ {+ "filename": "dead-letter-queue.md",+ "rank": 5,+ "score": 3.5010269,+ "snippet_found": false+ },+ {+ "filename": "delivery-retries.md",+ "rank": 6,+ "score": 3.4314427,+ "snippet_found": false+ },+ {+ "filename": "event-filtering.md",+ "rank": 7,+ "score": 3.1986198,+ "snippet_found": false+ },+ {+ "filename": "transformations.md",+ "rank": 8,+ "score": 3.1823845,+ "snippet_found": false+ },+ {+ "filename": "signing-secrets.md",+ "rank": 9,+ "score": 3.1128054,+ "snippet_found": false+ },+ {+ "filename": "audit-log.md",+ "rank": 10,+ "score": 3.0919387,+ "snippet_found": false+ },+ {+ "filename": "configuration-reference.md",+ "rank": 11,+ "score": 2.918428,+ "snippet_found": false+ },+ {+ "filename": "monitoring-and-alerts.md",+ "rank": 12,+ "score": 2.7586346,+ "snippet_found": false+ },+ {+ "filename": "troubleshooting.md",+ "rank": 13,+ "score": 2.7350254,+ "snippet_found": false+ },+ {+ "filename": "regions.md",+ "rank": 14,+ "score": 2.7063363,+ "snippet_found": false+ },+ {+ "filename": "single-sign-on.md",+ "rank": 15,+ "score": 2.550699,+ "snippet_found": false+ },+ {+ "filename": "rate-limits.md",+ "rank": 16,+ "score": 2.3362465,+ "snippet_found": false+ },+ {+ "filename": "invoices-and-payments.md",+ "rank": 17,+ "score": 2.2635229,+ "snippet_found": false+ },+ {+ "filename": "roles-and-permissions.md",+ "rank": 18,+ "score": 2.2386634,+ "snippet_found": false+ },+ {+ "filename": "billing-plans.md",+ "rank": 19,+ "score": 2.2386358,+ "snippet_found": false+ },+ {+ "filename": "api-keys.md",+ "rank": 20,+ "score": 2.1976867,+ "snippet_found": false+ }+ ],+ "expected_ranks": {+ "getting-started.md": 1,+ "error-codes.md": 2+ },+ "snippet_found": true,+ "passage_rank": 1,+ "metrics": {+ "recall@1": 0.5,+ "recall@3": 1.0,+ "recall@5": 1.0,+ "recall@10": 1.0,+ "reciprocal_rank": 1.0,+ "ndcg@1": 1.0,+ "ndcg@3": 1.0,+ "ndcg@5": 1.0,+ "ndcg@10": 1.0,+ "passage_hit@1": 1.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_hit@10": 1.0,+ "passage_rr": 1.0+ }+ }+ ]+}