Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
296 KB
Raw Blame
{
  "schema_version": 1,
  "run": {
    "run_id": "eval-49debd2f",
    "timestamp": "2026-11-05T18:03:57Z",
    "duration_s": 85.9,
    "target_url": "http://localhost:3000",
    "sdk_version": "0.4.0.dev0",
    "git_commit": "a7076dc551c4b940b63119b9c242665fa927f880",
    "corpus_files": 21,
    "corpus_hash": "9a346ec3274599771b53573f708fca382c23a27b19d102a84210377df06ab73a",
    "query_count": 83,
    "passage_questions": 77,
    "queries_hash": "a76d3448549d17810642d5a5206a56f54b8e8f605b43def0d3791941ba9f31bf",
    "k_values": [
      1,
      3,
      5,
      10
    ],
    "primary_k": 5,
    "fetch_limit": 50,
    "score_threshold": 0.0
  },
  "settings": {
    "embedding_provider": "azure",
    "embedding_model": "text-embedding-3-small",
    "chunk_size": 1000,
    "chunk_overlap": 200,
    "table_structure": true,
    "ocr": false,
    "picture_descriptions": false
  },
  "settings_note": null,
  "metrics": {
    "overall": {
      "recall@1": 0.6265060240963856,
      "recall@3": 0.8493975903614458,
      "recall@5": 0.9096385542168675,
      "recall@10": 0.963855421686747,
      "mrr": 0.7938898450946643,
      "ndcg@1": 0.7108433734939759,
      "ndcg@3": 0.7762517657398734,
      "ndcg@5": 0.8030453462731483,
      "ndcg@10": 0.8211616182694974,
      "passage_hit@1": 0.5324675324675324,
      "passage_hit@3": 0.7012987012987013,
      "passage_hit@5": 0.7922077922077922,
      "passage_hit@10": 0.8701298701298701,
      "passage_mrr": 0.6384012790334507
    },
    "by_tag": {
      "exact-token": {
        "count": 13,
        "passage_count": 13,
        "recall@1": 0.7307692307692307,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "mrr": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.9876493521766443,
        "ndcg@5": 0.9876493521766443,
        "ndcg@10": 0.9876493521766443,
        "passage_hit@1": 0.8461538461538461,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_mrr": 0.9230769230769231
      },
      "keyword": {
        "count": 7,
        "passage_count": 6,
        "recall@1": 0.5714285714285714,
        "recall@3": 0.9285714285714286,
        "recall@5": 0.9285714285714286,
        "recall@10": 1.0,
        "mrr": 0.8333333333333333,
        "ndcg@1": 0.7142857142857143,
        "ndcg@3": 0.8205824209052738,
        "ndcg@5": 0.8205824209052738,
        "ndcg@10": 0.8459023190172805,
        "passage_hit@1": 0.6666666666666666,
        "passage_hit@3": 0.8333333333333334,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_mrr": 0.7638888888888888
      },
      "multi-doc": {
        "count": 16,
        "passage_count": 11,
        "recall@1": 0.4375,
        "recall@3": 0.71875,
        "recall@5": 0.84375,
        "recall@10": 0.9375,
        "mrr": 0.903125,
        "ndcg@1": 0.875,
        "ndcg@3": 0.7440735963827292,
        "ndcg@5": 0.8050524251877592,
        "ndcg@10": 0.8413163264616205,
        "passage_hit@1": 0.7272727272727273,
        "passage_hit@3": 0.9090909090909091,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_mrr": 0.8257575757575758
      },
      "semantic": {
        "count": 63,
        "passage_count": 58,
        "recall@1": 0.6111111111111112,
        "recall@3": 0.8095238095238095,
        "recall@5": 0.8888888888888888,
        "recall@10": 0.9523809523809523,
        "mrr": 0.7469765684051398,
        "ndcg@1": 0.6507936507936508,
        "ndcg@3": 0.7277044132027969,
        "ndcg@5": 0.7630038923180636,
        "ndcg@10": 0.7840580397611262,
        "passage_hit@1": 0.4482758620689655,
        "passage_hit@3": 0.6206896551724138,
        "passage_hit@5": 0.7241379310344828,
        "passage_hit@10": 0.8275862068965517,
        "passage_mrr": 0.5616131922800408
      }
    }
  },
  "queries": [
    {
      "id": "key-rotate",
      "query": "How do I swap out an API key without causing downtime?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "api-keys.md"
      ],
      "expected_text": "overlap window",
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "api-keys.md",
          "rank": 1,
          "score": 4.485182,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.1399813,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 3.9088736,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 4,
          "score": 3.9022071,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.8871171,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 6,
          "score": 3.7825556,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 7,
          "score": 3.5549948,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 3.3526278,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 3.1453872,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 3.0133064,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 2.9531522,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 12,
          "score": 2.9356976,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 2.8147604,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 2.790321,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 2.6110516,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 16,
          "score": 2.5505896,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.3984993,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "api-keys.md": 1
      },
      "snippet_found": true,
      "passage_rank": 10,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.1
      }
    },
    {
      "id": "key-leaked",
      "query": "One of our keys was pushed to a public repo, what should we do right now?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "api-keys.md"
      ],
      "expected_text": "Revoke immediately",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "api-keys.md",
          "rank": 1,
          "score": 4.5577087,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 2,
          "score": 4.4239907,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 3.8773596,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 4,
          "score": 3.7586002,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 5,
          "score": 3.7468228,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 6,
          "score": 3.6312208,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 7,
          "score": 3.373435,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 8,
          "score": 3.268697,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 9,
          "score": 3.2684705,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 3.224593,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 3.1721559,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 3.1261327,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 13,
          "score": 3.0540926,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 14,
          "score": 3.023003,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.9438276,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 16,
          "score": 2.9110684,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 2.8642788,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 2.65653,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 19,
          "score": 2.5132947,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "api-keys.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "key-scopes",
      "query": "What scopes can I give an API key?",
      "tags": [
        "keyword"
      ],
      "expected_files": [
        "api-keys.md"
      ],
      "expected_text": "destinations:manage",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "api-keys.md",
          "rank": 1,
          "score": 5.5974355,
          "snippet_found": true
        },
        {
          "filename": "error-codes.md",
          "rank": 2,
          "score": 5.0317492,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 3,
          "score": 4.3466816,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 3.7902362,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.6450343,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 6,
          "score": 3.5535414,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 7,
          "score": 3.533179,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 8,
          "score": 3.477205,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 9,
          "score": 3.3471954,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 3.334623,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 3.179077,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 12,
          "score": 3.147795,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 13,
          "score": 3.0607226,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 14,
          "score": 2.8921738,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.697281,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 2.5704966,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 2.479469,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 18,
          "score": 2.3684938,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 19,
          "score": 2.3103514,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "api-keys.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "key-prefix-ci",
      "query": "What is a key starting with rwk_ci_ used for?",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "api-keys.md"
      ],
      "expected_text": "Automation key",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "api-keys.md",
          "rank": 1,
          "score": 5.3789306,
          "snippet_found": true
        },
        {
          "filename": "cli-reference.md",
          "rank": 2,
          "score": 4.635419,
          "snippet_found": true
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 3.814974,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 4,
          "score": 3.5494313,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 5,
          "score": 3.0389695,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 6,
          "score": 2.91465,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 2.8971639,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 2.8961177,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 9,
          "score": 2.875177,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.8743634,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 2.7834806,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 2.7593927,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 13,
          "score": 2.5931182,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 14,
          "score": 2.5388367,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 2.5265174,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 2.4976048,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.3593998,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 18,
          "score": 2.2960348,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 19,
          "score": 2.2928317,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 20,
          "score": 2.171982,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "api-keys.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "sig-verify",
      "query": "How can my server confirm that a webhook really came from Relaywise and was not forged?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "signing-secrets.md"
      ],
      "expected_text": "HMAC-SHA256",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "signing-secrets.md",
          "rank": 1,
          "score": 5.4620786,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 2,
          "score": 5.1428657,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 5.107025,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 4,
          "score": 4.8653693,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 5,
          "score": 4.459428,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 6,
          "score": 4.323626,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 4.092972,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 8,
          "score": 4.032093,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 9,
          "score": 3.9983268,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 10,
          "score": 3.9548411,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 11,
          "score": 3.8796291,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 12,
          "score": 3.8189056,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 3.8055673,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 14,
          "score": 3.7612507,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 15,
          "score": 3.748525,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 3.7323694,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 3.6424484,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 18,
          "score": 3.5377412,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 19,
          "score": 3.460004,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 20,
          "score": 3.3449996,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "signing-secrets.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "sig-roll",
      "query": "How do I change the secret used to sign deliveries, and how long do the old and new ones both work?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "signing-secrets.md"
      ],
      "expected_text": "72 hours",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "audit-log.md",
          "rank": 1,
          "score": 8.1549225,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 7.6038017,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 3,
          "score": 7.401187,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 6.34908,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 6.135564,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 6,
          "score": 5.6227202,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 7,
          "score": 5.285435,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 5.0490365,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 9,
          "score": 4.9172583,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 10,
          "score": 4.916431,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 11,
          "score": 4.8609548,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 12,
          "score": 4.823242,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 13,
          "score": 4.7858067,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 14,
          "score": 4.7696395,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 4.65145,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 4.4600706,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 17,
          "score": 4.159475,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 18,
          "score": 3.9943762,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 19,
          "score": 3.8904157,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 20,
          "score": 3.7599213,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "signing-secrets.md": 3
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "sig-max-age",
      "query": "How old can a signed request be before I should refuse it?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "signing-secrets.md"
      ],
      "expected_text": "five minutes",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "signing-secrets.md",
          "rank": 1,
          "score": 5.4995303,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 5.3574557,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 4.620363,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 4,
          "score": 4.157878,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 5,
          "score": 3.9991243,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 6,
          "score": 3.7961216,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 3.7712054,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 3.6903996,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 9,
          "score": 3.672656,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 3.3198228,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 11,
          "score": 3.3114727,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 3.3018017,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 13,
          "score": 3.2214346,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 3.219668,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 2.914604,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 2.8591857,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.8397179,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 18,
          "score": 2.7246382,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 19,
          "score": 2.6846876,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 20,
          "score": 2.6837616,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 21,
          "score": 2.5916595,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "signing-secrets.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "sig-whsec",
      "query": "Where do I find the whsec_ value for a destination?",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "signing-secrets.md"
      ],
      "expected_text": "Reveal",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "signing-secrets.md",
          "rank": 1,
          "score": 6.0598135,
          "snippet_found": true
        },
        {
          "filename": "configuration-reference.md",
          "rank": 2,
          "score": 4.84408,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 4.648713,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.378025,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.803798,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 6,
          "score": 3.4391446,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 3.3304148,
          "snippet_found": true
        },
        {
          "filename": "regions.md",
          "rank": 8,
          "score": 3.3192945,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 9,
          "score": 2.9882765,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 10,
          "score": 2.8431144,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 11,
          "score": 2.8061593,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 2.6899495,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 13,
          "score": 2.6079462,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 14,
          "score": 2.5567536,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 2.555109,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 2.4441526,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 17,
          "score": 2.355443,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 18,
          "score": 2.2208116,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 19,
          "score": 2.1636539,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "signing-secrets.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "sso-setup",
      "query": "How do we let staff log in to the console with their company accounts?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "single-sign-on.md"
      ],
      "expected_text": "identity provider",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "audit-log.md",
          "rank": 1,
          "score": 4.916114,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 2,
          "score": 4.913085,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 4.8817015,
          "snippet_found": true
        },
        {
          "filename": "cli-reference.md",
          "rank": 4,
          "score": 4.7266417,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 5,
          "score": 4.4032316,
          "snippet_found": true
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 6,
          "score": 4.382999,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 7,
          "score": 4.0702477,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 8,
          "score": 4.055292,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 3.9624317,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 10,
          "score": 3.7526836,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 11,
          "score": 3.4365666,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 12,
          "score": 3.0746837,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 13,
          "score": 3.007635,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 14,
          "score": 2.8946323,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 2.8831072,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 2.7998972,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.716467,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 18,
          "score": 2.7119622,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 19,
          "score": 2.5842686,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 20,
          "score": 2.5467687,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "single-sign-on.md": 5
      },
      "snippet_found": true,
      "passage_rank": 6,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.2,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.38685280723454163,
        "ndcg@10": 0.38685280723454163,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.16666666666666666
      }
    },
    {
      "id": "sso-lockout",
      "query": "If our identity provider goes down, can anyone still get into the console?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "single-sign-on.md"
      ],
      "expected_text": "break-glass",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "single-sign-on.md",
          "rank": 1,
          "score": 7.240963,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 3.965464,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 3.9560423,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 4,
          "score": 3.5496902,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 5,
          "score": 3.2137072,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 6,
          "score": 3.2129083,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 7,
          "score": 3.2106323,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.1655471,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 9,
          "score": 3.0810347,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 10,
          "score": 3.0189486,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 2.8896675,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 2.8698277,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 2.6514459,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 14,
          "score": 2.6178923,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 2.6146202,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 2.5582738,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 17,
          "score": 2.5566497,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 2.5499167,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 19,
          "score": 2.1304684,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "single-sign-on.md": 1
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "sso-offboarding",
      "query": "How is someone's access removed automatically when they leave the company?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "single-sign-on.md"
      ],
      "expected_text": "SCIM",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "single-sign-on.md",
          "rank": 1,
          "score": 5.3100443,
          "snippet_found": true
        },
        {
          "filename": "signing-secrets.md",
          "rank": 2,
          "score": 4.564142,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 3,
          "score": 4.112088,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 4,
          "score": 4.080737,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 5,
          "score": 3.9498796,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 6,
          "score": 3.8103957,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 7,
          "score": 3.748855,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 3.6031716,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 9,
          "score": 3.5195658,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 3.4794078,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 11,
          "score": 3.4275222,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 12,
          "score": 3.338546,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 13,
          "score": 3.278238,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 3.2054431,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 15,
          "score": 3.070385,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 16,
          "score": 2.9049387,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 17,
          "score": 2.4249506,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 18,
          "score": 2.311882,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 19,
          "score": 2.2950606,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 20,
          "score": 2.2608783,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "single-sign-on.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "session-length",
      "query": "How long before the console logs me out?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "single-sign-on.md",
        "configuration-reference.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "network-security.md",
          "rank": 1,
          "score": 4.718949,
          "snippet_found": null
        },
        {
          "filename": "audit-log.md",
          "rank": 2,
          "score": 4.631812,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 3,
          "score": 4.393179,
          "snippet_found": null
        },
        {
          "filename": "transformations.md",
          "rank": 4,
          "score": 3.361267,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 3.2693686,
          "snippet_found": null
        },
        {
          "filename": "single-sign-on.md",
          "rank": 6,
          "score": 3.2635205,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 7,
          "score": 3.2346013,
          "snippet_found": null
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 8,
          "score": 3.1757994,
          "snippet_found": null
        },
        {
          "filename": "billing-plans.md",
          "rank": 9,
          "score": 3.057241,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 10,
          "score": 2.9434013,
          "snippet_found": null
        },
        {
          "filename": "rate-limits.md",
          "rank": 11,
          "score": 2.8661394,
          "snippet_found": null
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 12,
          "score": 2.8531485,
          "snippet_found": null
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 2.7566361,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 14,
          "score": 2.6207955,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 15,
          "score": 2.4442549,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 16,
          "score": 2.4371514,
          "snippet_found": null
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 2.3959312,
          "snippet_found": null
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 2.392912,
          "snippet_found": null
        },
        {
          "filename": "api-keys.md",
          "rank": 19,
          "score": 2.146281,
          "snippet_found": null
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 20,
          "score": 1.9371018,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "single-sign-on.md": 6,
        "configuration-reference.md": 5
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.5,
        "recall@10": 1.0,
        "reciprocal_rank": 0.2,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.23719771276929622,
        "ndcg@10": 0.45560514958746035
      }
    },
    {
      "id": "dlq-purge-who",
      "query": "Who is allowed to empty a dead-letter queue?",
      "tags": [
        "keyword",
        "multi-doc"
      ],
      "expected_files": [
        "roles-and-permissions.md",
        "dead-letter-queue.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 5.936977,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 2,
          "score": 5.469982,
          "snippet_found": null
        },
        {
          "filename": "api-keys.md",
          "rank": 3,
          "score": 4.6652865,
          "snippet_found": null
        },
        {
          "filename": "troubleshooting.md",
          "rank": 4,
          "score": 4.4582715,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 5,
          "score": 4.1814966,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 6,
          "score": 4.075127,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 7,
          "score": 4.028639,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 8,
          "score": 3.9006016,
          "snippet_found": null
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 3.8686628,
          "snippet_found": null
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 10,
          "score": 3.857781,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 11,
          "score": 3.807433,
          "snippet_found": null
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 3.4361827,
          "snippet_found": null
        },
        {
          "filename": "network-security.md",
          "rank": 13,
          "score": 2.9324346,
          "snippet_found": null
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 2.883803,
          "snippet_found": null
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 2.7358198,
          "snippet_found": null
        },
        {
          "filename": "rate-limits.md",
          "rank": 16,
          "score": 2.616968,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 17,
          "score": 2.392273,
          "snippet_found": null
        },
        {
          "filename": "event-filtering.md",
          "rank": 18,
          "score": 2.3337693,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "roles-and-permissions.md": 10,
        "dead-letter-queue.md": 1
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 0.5,
        "recall@5": 0.5,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.6131471927654584,
        "ndcg@5": 0.6131471927654584,
        "ndcg@10": 0.7903864795495061
      }
    },
    {
      "id": "viewer-payloads",
      "query": "Can a read-only user see the contents of events?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "roles-and-permissions.md"
      ],
      "expected_text": "Payload access",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "roles-and-permissions.md",
          "rank": 1,
          "score": 4.9205885,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.8499303,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 3,
          "score": 4.828339,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.6417646,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 5,
          "score": 4.445083,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 6,
          "score": 4.2635155,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 7,
          "score": 3.988793,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 8,
          "score": 3.8740609,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 3.845269,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 3.5629725,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 3.4642334,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 3.359555,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 13,
          "score": 3.3335032,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 14,
          "score": 3.3266916,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 3.1584954,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 16,
          "score": 3.0837433,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 17,
          "score": 3.0490007,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 2.998888,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 19,
          "score": 2.7927902,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "roles-and-permissions.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "invite-expiry",
      "query": "How long does a team invitation link stay valid?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "roles-and-permissions.md"
      ],
      "expected_text": "valid for 7 days",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "roles-and-permissions.md",
          "rank": 1,
          "score": 5.6239557,
          "snippet_found": true
        },
        {
          "filename": "audit-log.md",
          "rank": 2,
          "score": 4.895487,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 3,
          "score": 4.1866374,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 4.109663,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 5,
          "score": 4.0150146,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 6,
          "score": 3.691551,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 3.5175457,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 3.4534285,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 9,
          "score": 3.2989433,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 3.2002692,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 11,
          "score": 2.9563212,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 12,
          "score": 2.823382,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 2.8055763,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 14,
          "score": 2.7686763,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 2.7495055,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 16,
          "score": 2.7170486,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 17,
          "score": 2.7069454,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 2.6110837,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 19,
          "score": 2.4631553,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 20,
          "score": 2.0309317,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 21,
          "score": 2.0022187,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "roles-and-permissions.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "ownership-transfer",
      "query": "How do I hand a workspace over to a colleague?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "roles-and-permissions.md"
      ],
      "expected_text": "last Owner",
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 4.235373,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 2,
          "score": 4.2214937,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 3,
          "score": 4.050256,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 4,
          "score": 3.4798121,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 5,
          "score": 3.3237166,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 3.3111908,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 7,
          "score": 3.0920124,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 8,
          "score": 3.0649781,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 2.9621043,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.6565669,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 11,
          "score": 2.6309865,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 2.5766845,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 2.551534,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 2.524885,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 15,
          "score": 2.5160358,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 16,
          "score": 2.360861,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 17,
          "score": 2.2255254,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "roles-and-permissions.md": 4
      },
      "snippet_found": true,
      "passage_rank": 5,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.25,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.43067655807339306,
        "ndcg@10": 0.43067655807339306,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.2
      }
    },
    {
      "id": "retry-schedule",
      "query": "How many times will a failed webhook be attempted, and over how long?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "delivery-retries.md"
      ],
      "expected_text": "27 hours",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "delivery-retries.md",
          "rank": 1,
          "score": 5.953916,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 5.9165845,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 3,
          "score": 5.629542,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 5.3382273,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 5,
          "score": 5.2244463,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 6,
          "score": 4.7950573,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 4.6740932,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 8,
          "score": 4.488102,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 4.288582,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 10,
          "score": 4.247769,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 11,
          "score": 4.176223,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 12,
          "score": 4.0876427,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 3.7292528,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 14,
          "score": 3.6605527,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 15,
          "score": 3.4551523,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 16,
          "score": 3.4263456,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 17,
          "score": 3.2239738,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 2.5783687,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 19,
          "score": 2.4560182,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 1
      },
      "snippet_found": true,
      "passage_rank": 6,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.16666666666666666
      }
    },
    {
      "id": "cfg-max-attempts",
      "query": "What is the default for delivery.max_attempts?",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "delivery-retries.md",
        "configuration-reference.md"
      ],
      "expected_text": "delivery.max_attempts",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "delivery-retries.md",
          "rank": 1,
          "score": 4.100553,
          "snippet_found": true
        },
        {
          "filename": "configuration-reference.md",
          "rank": 2,
          "score": 3.990864,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 3,
          "score": 2.4812531,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 2.3984542,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 5,
          "score": 2.262816,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 6,
          "score": 2.2063332,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 7,
          "score": 2.1881351,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 8,
          "score": 2.1569738,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 9,
          "score": 2.0785315,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.0601966,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 11,
          "score": 2.0537386,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 12,
          "score": 2.0393662,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 13,
          "score": 2.037712,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 14,
          "score": 1.9448432,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 15,
          "score": 1.8971537,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 1.8890164,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 17,
          "score": 1.8565283,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 18,
          "score": 1.8449395,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 19,
          "score": 1.8378721,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 20,
          "score": 1.8362129,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 1,
        "configuration-reference.md": 2
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "circuit-breaker",
      "query": "What happens when most requests to one endpoint start failing?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "delivery-retries.md"
      ],
      "expected_text": "degraded",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 1,
          "score": 5.0543203,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 2,
          "score": 4.903905,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 4.752765,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.5258617,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 5,
          "score": 4.1057053,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 6,
          "score": 4.084237,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 7,
          "score": 3.9673247,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.947473,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 3.8320982,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 10,
          "score": 3.7889988,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 11,
          "score": 3.705308,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 12,
          "score": 3.6559482,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 13,
          "score": 3.598694,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 14,
          "score": 3.2583046,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 15,
          "score": 3.1484778,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 16,
          "score": 2.9071856,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 2.8332481,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 2.7670798,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 19,
          "score": 2.5437899,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 2
      },
      "snippet_found": true,
      "passage_rank": 2,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.5,
        "ndcg@1": 0.0,
        "ndcg@3": 0.6309297535714575,
        "ndcg@5": 0.6309297535714575,
        "ndcg@10": 0.6309297535714575,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.5
      }
    },
    {
      "id": "gone-410",
      "query": "What does Relaywise do if my endpoint answers 410 Gone?",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "delivery-retries.md",
        "dead-letter-queue.md"
      ],
      "expected_text": "410 Gone",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "delivery-retries.md",
          "rank": 1,
          "score": 5.6472993,
          "snippet_found": true
        },
        {
          "filename": "error-codes.md",
          "rank": 2,
          "score": 4.9761944,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 4.925442,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 4,
          "score": 4.794298,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 5,
          "score": 4.408581,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 6,
          "score": 4.156478,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 7,
          "score": 4.053644,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 3.842249,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 9,
          "score": 3.769952,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 10,
          "score": 3.6307647,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 3.591357,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 12,
          "score": 3.158509,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 13,
          "score": 3.0485291,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 14,
          "score": 2.950628,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.7675254,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 2.7243469,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.7232513,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 18,
          "score": 2.5460482,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 19,
          "score": 2.5377195,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 20,
          "score": 2.4653406,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 1,
        "dead-letter-queue.md": 3
      },
      "snippet_found": true,
      "passage_rank": 2,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.9197207891481876,
        "ndcg@5": 0.9197207891481876,
        "ndcg@10": 0.9197207891481876,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.5
      }
    },
    {
      "id": "duplicate-events",
      "query": "Why did my endpoint get the same event twice?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "delivery-retries.md",
        "troubleshooting.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 6.117975,
          "snippet_found": null
        },
        {
          "filename": "event-filtering.md",
          "rank": 2,
          "score": 5.701635,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 4.079763,
          "snippet_found": null
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 3.9514606,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 3.9501805,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 6,
          "score": 3.8345904,
          "snippet_found": null
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 3.5684028,
          "snippet_found": null
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 3.4548547,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 3.4442132,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 3.2959082,
          "snippet_found": null
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 3.278033,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 3.2729166,
          "snippet_found": null
        },
        {
          "filename": "transformations.md",
          "rank": 13,
          "score": 3.2519,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 3.0763547,
          "snippet_found": null
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 15,
          "score": 2.9944477,
          "snippet_found": null
        },
        {
          "filename": "network-security.md",
          "rank": 16,
          "score": 2.795557,
          "snippet_found": null
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 2.4639845,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 5,
        "troubleshooting.md": 1
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 0.5,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.6131471927654584,
        "ndcg@5": 0.8503449055347546,
        "ndcg@10": 0.8503449055347546
      }
    },
    {
      "id": "ordered-delivery",
      "query": "Can I make sure events for one customer are delivered in sequence?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "delivery-retries.md"
      ],
      "expected_text": "ordering key",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 5.0055847,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 2,
          "score": 4.8828163,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 3,
          "score": 4.750148,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 4.722822,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 5,
          "score": 4.638679,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 6,
          "score": 4.568696,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 4.3880367,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 4.3245573,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 9,
          "score": 4.2667556,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 10,
          "score": 4.13976,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 11,
          "score": 4.1162443,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 4.1093607,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 4.1089215,
          "snippet_found": true
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 4.0251784,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 3.751133,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 16,
          "score": 3.4920778,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 3.3043363,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 3.2784362,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 3
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "destination-auto-disabled",
      "query": "Why was my destination turned off without anyone touching it?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "delivery-retries.md",
        "troubleshooting.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.720254,
          "snippet_found": null
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 4.6855354,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 4.230507,
          "snippet_found": null
        },
        {
          "filename": "audit-log.md",
          "rank": 4,
          "score": 4.065977,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 3.9990711,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 6,
          "score": 3.6033638,
          "snippet_found": null
        },
        {
          "filename": "network-security.md",
          "rank": 7,
          "score": 3.3606205,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 3.3185124,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 9,
          "score": 3.0583892,
          "snippet_found": null
        },
        {
          "filename": "event-filtering.md",
          "rank": 10,
          "score": 2.7916946,
          "snippet_found": null
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 2.7792022,
          "snippet_found": null
        },
        {
          "filename": "signing-secrets.md",
          "rank": 12,
          "score": 2.6849184,
          "snippet_found": null
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 13,
          "score": 2.385123,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 14,
          "score": 2.2865496,
          "snippet_found": null
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 2.131452,
          "snippet_found": null
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.1214397,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 2.0443506,
          "snippet_found": null
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 1.9859804,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "delivery-retries.md": 5,
        "troubleshooting.md": 1
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 0.5,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.6131471927654584,
        "ndcg@5": 0.8503449055347546,
        "ndcg@10": 0.8503449055347546
      }
    },
    {
      "id": "dlq-what",
      "query": "Where do events go once every delivery attempt has been used up?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "dead-letter-queue.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 7.40966,
          "snippet_found": null
        },
        {
          "filename": "rate-limits.md",
          "rank": 2,
          "score": 6.4829597,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 3,
          "score": 6.2448034,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 4,
          "score": 5.8351073,
          "snippet_found": null
        },
        {
          "filename": "event-filtering.md",
          "rank": 5,
          "score": 5.679214,
          "snippet_found": null
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 5.4656425,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 5.4533553,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 8,
          "score": 5.332749,
          "snippet_found": null
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 4.9943905,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 4.968727,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 4.9405565,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 4.714102,
          "snippet_found": null
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 4.632801,
          "snippet_found": null
        },
        {
          "filename": "network-security.md",
          "rank": 14,
          "score": 4.599147,
          "snippet_found": null
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 4.52356,
          "snippet_found": null
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 4.458351,
          "snippet_found": null
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 17,
          "score": 4.3109264,
          "snippet_found": null
        },
        {
          "filename": "api-keys.md",
          "rank": 18,
          "score": 3.9744198,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "dead-letter-queue.md": 1
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0
      }
    },
    {
      "id": "dlq-retention-team",
      "query": "On the Team plan, how long are undeliverable events kept before they are removed?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "dead-letter-queue.md"
      ],
      "expected_text": "14 days",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "data-retention.md",
          "rank": 1,
          "score": 7.8197613,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 2,
          "score": 7.3345337,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 3,
          "score": 7.332274,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 6.697984,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 5.410752,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 6,
          "score": 5.0231695,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 4.924449,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 4.6471853,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 9,
          "score": 4.3576174,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 10,
          "score": 4.2551737,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 4.2262077,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 12,
          "score": 4.0809183,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 3.9892564,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 14,
          "score": 3.9197059,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 15,
          "score": 3.8503838,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 16,
          "score": 3.7696798,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 17,
          "score": 3.5610816,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 3.4350631,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "dead-letter-queue.md": 4
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.25,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.43067655807339306,
        "ndcg@10": 0.43067655807339306,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "dlq-replay-other-destination",
      "query": "How do I use rwctl dlq replay to resend failed events to a new endpoint?",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "dead-letter-queue.md"
      ],
      "expected_text": "dst_9x1aa",
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 8.870468,
          "snippet_found": true
        },
        {
          "filename": "cli-reference.md",
          "rank": 2,
          "score": 6.881461,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 5.751782,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 4,
          "score": 5.043284,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 5,
          "score": 4.7557015,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 4.6869016,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 7,
          "score": 4.359108,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 8,
          "score": 4.1611156,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 9,
          "score": 4.0299006,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 3.9948087,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 11,
          "score": 3.844854,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 3.5940793,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 13,
          "score": 3.408236,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 3.3590999,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 3.3468688,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 3.2550454,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 17,
          "score": 3.1481986,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "dead-letter-queue.md": 1
      },
      "snippet_found": true,
      "passage_rank": 2,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.5
      }
    },
    {
      "id": "replay-header",
      "query": "How can my endpoint tell that a delivery is a replay?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "dead-letter-queue.md"
      ],
      "expected_text": "Relaywise-Replay",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 5.8395176,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 5.7776,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 4.659917,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 4,
          "score": 4.5585823,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 4.535796,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 6,
          "score": 4.4854813,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 7,
          "score": 4.418273,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 4.399255,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 4.092454,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 10,
          "score": 3.7910876,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 11,
          "score": 3.6927078,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 3.5761201,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 3.4670155,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 14,
          "score": 3.373952,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 3.302986,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 3.2588396,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 3.145359,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 18,
          "score": 3.0399675,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "dead-letter-queue.md": 1
      },
      "snippet_found": true,
      "passage_rank": 10,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.1
      }
    },
    {
      "id": "ingest-rate",
      "query": "How fast am I allowed to send events on each plan?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "rate-limits.md"
      ],
      "expected_text": "events per second",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "rate-limits.md",
          "rank": 1,
          "score": 6.559215,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 2,
          "score": 6.377366,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 6.3756666,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 4,
          "score": 5.5507555,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 5,
          "score": 4.7628126,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 6,
          "score": 4.5803623,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 7,
          "score": 4.3698053,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 8,
          "score": 4.1771226,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 9,
          "score": 4.1309533,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 10,
          "score": 4.102806,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 11,
          "score": 3.8547676,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 12,
          "score": 3.605217,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 13,
          "score": 3.5623853,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 3.453558,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 15,
          "score": 3.3093355,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 16,
          "score": 3.2472394,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 17,
          "score": 3.2431312,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 3.1259098,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "rate-limits.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "err-4290",
      "query": "RW-4290",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "rate-limits.md",
        "error-codes.md"
      ],
      "expected_text": "RW-4290",
      "chunks_returned": 8,
      "unique_files_returned": 2,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 5.104968,
          "snippet_found": true
        },
        {
          "filename": "rate-limits.md",
          "rank": 2,
          "score": 4.61705,
          "snippet_found": true
        }
      ],
      "expected_ranks": {
        "rate-limits.md": 2,
        "error-codes.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "payload-max-size",
      "query": "What is the largest event body I can send?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "rate-limits.md"
      ],
      "expected_text": "256 KB",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.1728625,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 2,
          "score": 4.963256,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 4.2940288,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 3.8478184,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 5,
          "score": 3.7793765,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 6,
          "score": 3.7538767,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 3.7113836,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 8,
          "score": 3.647768,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 9,
          "score": 3.63453,
          "snippet_found": true
        },
        {
          "filename": "signing-secrets.md",
          "rank": 10,
          "score": 3.572057,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 3.561708,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 12,
          "score": 3.4531813,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 3.378756,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 14,
          "score": 3.3345623,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 3.2072341,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 16,
          "score": 3.0846722,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.9861572,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 18,
          "score": 2.791448,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 2.7770343,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "rate-limits.md": 9
      },
      "snippet_found": true,
      "passage_rank": 14,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.1111111111111111,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.0,
        "ndcg@10": 0.3010299956639812,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.07142857142857142
      }
    },
    {
      "id": "batch-size",
      "query": "How many events fit in one batch request?",
      "tags": [
        "keyword"
      ],
      "expected_files": [
        "rate-limits.md"
      ],
      "expected_text": "500 events",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "rate-limits.md",
          "rank": 1,
          "score": 6.440647,
          "snippet_found": true
        },
        {
          "filename": "event-filtering.md",
          "rank": 2,
          "score": 4.471184,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 3,
          "score": 4.471135,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 4.4559145,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 5,
          "score": 4.4337754,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 6,
          "score": 4.210376,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 7,
          "score": 4.1672397,
          "snippet_found": true
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.9377017,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 9,
          "score": 3.857754,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 10,
          "score": 3.7983751,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 3.741257,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 3.216155,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 13,
          "score": 3.2107887,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 14,
          "score": 3.144632,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 15,
          "score": 2.9834402,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 16,
          "score": 2.9600403,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 2.867922,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 18,
          "score": 2.8529837,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 19,
          "score": 2.5421212,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "rate-limits.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "err-3002",
      "query": "What does error RW-3002 mean?",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "error-codes.md"
      ],
      "expected_text": "TLS handshake failed",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 7.1374054,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 4.6004024,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 3,
          "score": 3.5720081,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 4,
          "score": 3.5708163,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 3.373531,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 3.3219764,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 7,
          "score": 3.270485,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 8,
          "score": 3.0856066,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 2.9129028,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 2.8195572,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 11,
          "score": 2.7496767,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 12,
          "score": 2.6049972,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 2.339275,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 14,
          "score": 2.285453,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 15,
          "score": 2.267887,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 16,
          "score": 2.1199887,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 1.9273345,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 1.7775464,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 1.6508594,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "error-codes.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "err-5004",
      "query": "RW-5004",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "error-codes.md",
        "transformations.md"
      ],
      "expected_text": "RW-5004",
      "chunks_returned": 8,
      "unique_files_returned": 2,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 7.083034,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 4.1704693,
          "snippet_found": true
        }
      ],
      "expected_ranks": {
        "error-codes.md": 1,
        "transformations.md": 2
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "err-2031",
      "query": "Why was my destination URL rejected with RW-2031?",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "error-codes.md",
        "network-security.md"
      ],
      "expected_text": "RW-2031",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 7.7312093,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 5.75021,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 4.6584964,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.269464,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 4.031956,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 3.6961586,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 3.5220945,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 8,
          "score": 3.4944685,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 9,
          "score": 3.3147504,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 3.3085356,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 11,
          "score": 3.0455709,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 3.013853,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 13,
          "score": 3.0125816,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 3.006339,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 15,
          "score": 2.862323,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.804049,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 2.4442608,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 18,
          "score": 2.4357595,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 2.4208562,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 20,
          "score": 2.021038,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "error-codes.md": 1,
        "network-security.md": 2
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "err-7001",
      "query": "What causes RW-7001?",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "error-codes.md",
        "regions.md"
      ],
      "expected_text": "RW-7001",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 4.55962,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 3.2900271,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 3,
          "score": 3.231194,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 3.0057564,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 2.8333864,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 6,
          "score": 2.6037674,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 2.4505215,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 8,
          "score": 2.2581377,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 9,
          "score": 2.2200274,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 2.1641731,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 2.1146564,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 2.0174422,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 1.6311694,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 14,
          "score": 1.4668834,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 15,
          "score": 1.456635,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 1.4466288,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 17,
          "score": 1.4390881,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 1.4314262,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 1.4135953,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "error-codes.md": 1,
        "regions.md": 3
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.9197207891481876,
        "ndcg@5": 0.9197207891481876,
        "ndcg@10": 0.9197207891481876,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "cfg-concurrency",
      "query": "Which setting limits how many requests hit my endpoint in parallel?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "configuration-reference.md"
      ],
      "expected_text": "delivery.max_concurrency",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "configuration-reference.md",
          "rank": 1,
          "score": 6.5323057,
          "snippet_found": true
        },
        {
          "filename": "rate-limits.md",
          "rank": 2,
          "score": 5.4767127,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 4.882698,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 4,
          "score": 4.4162087,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 4.319521,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 6,
          "score": 3.912937,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 7,
          "score": 3.9068096,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 8,
          "score": 3.5763764,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 9,
          "score": 3.4460037,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 10,
          "score": 3.442761,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 11,
          "score": 3.3418725,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 3.2560182,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 13,
          "score": 3.2276452,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 14,
          "score": 3.1950235,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 15,
          "score": 3.0526667,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 16,
          "score": 2.9480395,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 17,
          "score": 2.910755,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 2.5600786,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "configuration-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "cfg-dedupe-window",
      "query": "ingest.dedupe_window_seconds",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "configuration-reference.md"
      ],
      "expected_text": "ingest.dedupe_window_seconds",
      "chunks_returned": 8,
      "unique_files_returned": 1,
      "ranked_files": [
        {
          "filename": "configuration-reference.md",
          "rank": 1,
          "score": 3.756914,
          "snippet_found": true
        }
      ],
      "expected_ranks": {
        "configuration-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "cfg-explain",
      "query": "How do I find out which level a setting's value is coming from?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "configuration-reference.md",
        "cli-reference.md"
      ],
      "expected_text": "rwctl config explain",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "configuration-reference.md",
          "rank": 1,
          "score": 5.1910233,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 5.1194305,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 3,
          "score": 4.7757354,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 4,
          "score": 4.3601513,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 4.301612,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 6,
          "score": 4.265447,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 3.9866903,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 8,
          "score": 3.753189,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 3.693151,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 10,
          "score": 3.283983,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 3.2680726,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 12,
          "score": 3.261208,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 3.0610666,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 2.967939,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 15,
          "score": 2.8458738,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 16,
          "score": 2.8134208,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.7762527,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 18,
          "score": 2.6981857,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 2.6218233,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 20,
          "score": 2.5113726,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 21,
          "score": 2.4949827,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "configuration-reference.md": 1,
        "cli-reference.md": 8
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 0.5,
        "recall@5": 0.5,
        "recall@10": 0.5,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.6131471927654584,
        "ndcg@5": 0.6131471927654584,
        "ndcg@10": 0.6131471927654584,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "schema-validation",
      "query": "Can incoming payloads be checked against a schema and refused when they don't match?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "configuration-reference.md"
      ],
      "expected_text": "ingest.schema_validation",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "configuration-reference.md",
          "rank": 1,
          "score": 7.2857604,
          "snippet_found": true
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 2,
          "score": 4.1829777,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 3.9681184,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 3.9628394,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 3.8958888,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 6,
          "score": 3.8805716,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 3.7766762,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 8,
          "score": 3.7272916,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 3.6178882,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 10,
          "score": 3.5861323,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 11,
          "score": 3.5555618,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 12,
          "score": 3.5408092,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 13,
          "score": 3.2773092,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 3.2318847,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 15,
          "score": 3.1293461,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 16,
          "score": 3.0882106,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 17,
          "score": 2.9718976,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 18,
          "score": 2.8200388,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 19,
          "score": 2.7177334,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "configuration-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "filter-wildcard",
      "query": "How do I send every kind of invoice event, including nested types, to one endpoint?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "event-filtering.md"
      ],
      "expected_text": "invoice.**",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 6.599655,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 2,
          "score": 5.760765,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 3,
          "score": 5.569148,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 4,
          "score": 5.527859,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 5,
          "score": 5.472649,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 6,
          "score": 5.421536,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 5.261847,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 8,
          "score": 5.023038,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 9,
          "score": 4.937884,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 4.873948,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 4.553656,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 12,
          "score": 4.50159,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 13,
          "score": 4.4150305,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 4.278399,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 3.8844829,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 16,
          "score": 3.8552125,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 17,
          "score": 3.8130789,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 3.5139594,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 19,
          "score": 3.3147485,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "event-filtering.md": 7
      },
      "snippet_found": true,
      "passage_rank": 43,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.14285714285714285,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.0,
        "ndcg@10": 0.3333333333333333,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.023255813953488372
      }
    },
    {
      "id": "filter-missing-field",
      "query": "What happens when a filter refers to a field the event doesn't have?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "event-filtering.md"
      ],
      "expected_text": "it is not an error",
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "event-filtering.md",
          "rank": 1,
          "score": 6.3981233,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 6.132348,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 3,
          "score": 5.273807,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 4.9314632,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 4.871355,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 6,
          "score": 4.5469966,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 4.467554,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 4.247962,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 3.781559,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 10,
          "score": 3.587966,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 11,
          "score": 3.5403461,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 12,
          "score": 3.4803455,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 13,
          "score": 3.4259849,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 14,
          "score": 3.240863,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 3.217833,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 16,
          "score": 3.142176,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 17,
          "score": 3.012438,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "event-filtering.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "unrouted-events",
      "query": "What happens to an event that no route picks up?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "event-filtering.md"
      ],
      "expected_text": "unrouted",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "event-filtering.md",
          "rank": 1,
          "score": 6.7026176,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 6.576373,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 3,
          "score": 6.105928,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 5.184962,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 4.8381486,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 4.526141,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 4.250954,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 8,
          "score": 4.041794,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 4.0392456,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 3.9502084,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 3.8146112,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 3.8039114,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 13,
          "score": 3.4500139,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 14,
          "score": 3.4208,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 15,
          "score": 3.2320712,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 16,
          "score": 3.1246333,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 3.1182313,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 3.030007,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 19,
          "score": 2.9305377,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "event-filtering.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "route-fan-out",
      "query": "How many endpoints can one route deliver to?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "event-filtering.md"
      ],
      "expected_text": "up to 20 destinations",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 6.023015,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 2,
          "score": 5.745181,
          "snippet_found": true
        },
        {
          "filename": "configuration-reference.md",
          "rank": 3,
          "score": 5.1195517,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 4.844228,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 5,
          "score": 4.83197,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 6,
          "score": 4.238139,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 7,
          "score": 4.2260113,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 4.079329,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 3.8717012,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 10,
          "score": 3.8605428,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 11,
          "score": 3.7901826,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 12,
          "score": 3.5144677,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 13,
          "score": 3.5060587,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 14,
          "score": 3.367303,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 15,
          "score": 3.3218155,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 3.022322,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 17,
          "score": 2.9369514,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 18,
          "score": 2.922502,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 2.7017128,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "event-filtering.md": 2
      },
      "snippet_found": true,
      "passage_rank": 2,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.5,
        "ndcg@1": 0.0,
        "ndcg@3": 0.6309297535714575,
        "ndcg@5": 0.6309297535714575,
        "ndcg@10": 0.6309297535714575,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.5
      }
    },
    {
      "id": "strip-fields-before-delivery",
      "query": "How can I strip card numbers out of events before they are forwarded to a partner?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "transformations.md"
      ],
      "expected_text": "card_number",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 5.678779,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 2,
          "score": 4.841915,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 4.5332537,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 4.3789315,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 5,
          "score": 4.374852,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 6,
          "score": 4.303219,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 4.297907,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 4.2757826,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 9,
          "score": 4.0928054,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 10,
          "score": 4.013118,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 3.958882,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 12,
          "score": 3.8992069,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 13,
          "score": 3.830959,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 3.8095558,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 15,
          "score": 3.6439047,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 16,
          "score": 3.4828746,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 17,
          "score": 3.4731426,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 18,
          "score": 3.339641,
          "snippet_found": true
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 19,
          "score": 3.257053,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "transformations.md": 18
      },
      "snippet_found": true,
      "passage_rank": 44,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.0,
        "recall@10": 0.0,
        "reciprocal_rank": 0.05555555555555555,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.0,
        "ndcg@10": 0.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.022727272727272728
      }
    },
    {
      "id": "transform-network",
      "query": "Can transformation code call an external API?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "transformations.md"
      ],
      "expected_text": "No network access",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "error-codes.md",
          "rank": 1,
          "score": 4.3027563,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 2,
          "score": 3.7211428,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 3,
          "score": 3.5431662,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 3.4637728,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 5,
          "score": 3.4438903,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 6,
          "score": 3.3542087,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 7,
          "score": 3.1996362,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 3.1790874,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 9,
          "score": 3.1691957,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 10,
          "score": 2.7622945,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 11,
          "score": 2.7219043,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 12,
          "score": 2.7049937,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 2.6479259,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 14,
          "score": 2.5942392,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 15,
          "score": 2.5000453,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 2.412778,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 2.2223032,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 2.120767,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 2.0920844,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "transformations.md": 3
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "transform-rollback",
      "query": "I shipped a broken transformation, how do I go back to the previous one?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "transformations.md"
      ],
      "expected_text": "Make active",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.23054,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 2,
          "score": 4.2707195,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 3,
          "score": 4.0321445,
          "snippet_found": true
        },
        {
          "filename": "rate-limits.md",
          "rank": 4,
          "score": 3.5525649,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 3.4241276,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 6,
          "score": 3.4105673,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 3.4089398,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.4029415,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 3.3298483,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 3.2877781,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 11,
          "score": 3.2613475,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 3.2458348,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 3.2236872,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 14,
          "score": 3.2179403,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 15,
          "score": 3.1666298,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 16,
          "score": 2.8349867,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 2.7176538,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 18,
          "score": 2.5744977,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 2.480778,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "transformations.md": 3
      },
      "snippet_found": true,
      "passage_rank": 23,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.043478260869565216
      }
    },
    {
      "id": "plan-scale-price",
      "query": "What does the Scale plan cost and how many events come with it?",
      "tags": [
        "keyword"
      ],
      "expected_files": [
        "billing-plans.md"
      ],
      "expected_text": "20 million",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "data-retention.md",
          "rank": 1,
          "score": 6.165932,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 2,
          "score": 5.6396055,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 3,
          "score": 5.595186,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.96609,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 4.8011065,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 4.7937565,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 4.7090893,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 4.6951523,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 9,
          "score": 4.6432734,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 10,
          "score": 4.542534,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 11,
          "score": 3.9930236,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 3.9314911,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 3.7961717,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 14,
          "score": 3.7279844,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 3.7183979,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 16,
          "score": 3.5648823,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 17,
          "score": 3.5081198,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 18,
          "score": 3.369672,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "billing-plans.md": 3
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "over-quota",
      "query": "What happens if we send more events in a month than our plan includes?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "billing-plans.md"
      ],
      "expected_text": "billed as overage",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "billing-plans.md",
          "rank": 1,
          "score": 6.611384,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 5.573196,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 5.069791,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 4,
          "score": 4.896354,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 5,
          "score": 4.8308477,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 6,
          "score": 4.6927853,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 7,
          "score": 4.615057,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 8,
          "score": 4.568859,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 9,
          "score": 4.510277,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 4.4583926,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 4.3350053,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 4.3333607,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 13,
          "score": 4.2085896,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 3.864415,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 15,
          "score": 3.6676457,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 16,
          "score": 3.6663413,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 17,
          "score": 3.626868,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 18,
          "score": 3.5489278,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "billing-plans.md": 1
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "spending-cap",
      "query": "How do I make sure we never get a surprise bill?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "billing-plans.md"
      ],
      "expected_text": "hard cap",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "cli-reference.md",
          "rank": 1,
          "score": 4.176035,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 2,
          "score": 3.7238424,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 3,
          "score": 3.373507,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 3.3062384,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 3.1803827,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 2.8217483,
          "snippet_found": true
        },
        {
          "filename": "signing-secrets.md",
          "rank": 7,
          "score": 2.8194714,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 2.7950668,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 9,
          "score": 2.7918391,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 10,
          "score": 2.7697334,
          "snippet_found": true
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 11,
          "score": 2.3493757,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 12,
          "score": 2.2208354,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 13,
          "score": 2.1967921,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 14,
          "score": 2.0774598,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 15,
          "score": 2.0550961,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 16,
          "score": 2.0002537,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 17,
          "score": 1.9782275,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 18,
          "score": 1.8935627,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 19,
          "score": 1.8432963,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "billing-plans.md": 10
      },
      "snippet_found": true,
      "passage_rank": 11,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.1,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.0,
        "ndcg@10": 0.2890648263178879,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.09090909090909091
      }
    },
    {
      "id": "downgrade-timing",
      "query": "When does a plan downgrade kick in?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "billing-plans.md"
      ],
      "expected_text": "end of the current billing period",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "billing-plans.md",
          "rank": 1,
          "score": 4.5011973,
          "snippet_found": true
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 2,
          "score": 3.9330368,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 3.6819746,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 4,
          "score": 3.3807042,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 5,
          "score": 3.145202,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 6,
          "score": 2.9044037,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 2.6842203,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 8,
          "score": 2.652769,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 9,
          "score": 2.559432,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 10,
          "score": 2.5516438,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 2.5177975,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 12,
          "score": 2.4889445,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 2.4745228,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 14,
          "score": 2.397379,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 2.3737328,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.3688858,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 17,
          "score": 2.2727196,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 18,
          "score": 2.255967,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 19,
          "score": 2.1137109,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "billing-plans.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "nonprofit-discount",
      "query": "Is there a discount for nonprofits?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "billing-plans.md"
      ],
      "expected_text": "50 percent off",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 3.192665,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 2,
          "score": 3.1329155,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 2.7976594,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 4,
          "score": 2.500914,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 2.431467,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 6,
          "score": 2.3387825,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 7,
          "score": 2.1462817,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 2.138791,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 2.0935802,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 1.9763438,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 11,
          "score": 1.9572263,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 12,
          "score": 1.9540287,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 1.8847063,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 14,
          "score": 1.8813934,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 1.8805516,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 16,
          "score": 1.8092102,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 17,
          "score": 1.799421,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 1.7747855,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 1.7663033,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "billing-plans.md": 2
      },
      "snippet_found": true,
      "passage_rank": 2,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.5,
        "ndcg@1": 0.0,
        "ndcg@3": 0.6309297535714575,
        "ndcg@5": 0.6309297535714575,
        "ndcg@10": 0.6309297535714575,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.5
      }
    },
    {
      "id": "card-declined",
      "query": "What happens if our card is declined?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "invoices-and-payments.md"
      ],
      "expected_text": "suspended",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 4.5424986,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 3.1390238,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 3,
          "score": 2.9242349,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 2.853714,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 2.7698298,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 6,
          "score": 2.6912606,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 2.6068764,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 8,
          "score": 2.4017735,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 9,
          "score": 2.3534894,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 10,
          "score": 2.3194094,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 2.2925973,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 2.2087455,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 2.170509,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 14,
          "score": 2.1145196,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 2.0395908,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 16,
          "score": 1.8723415,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 17,
          "score": 1.8695025,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 18,
          "score": 1.8511734,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "invoices-and-payments.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "invoice-download",
      "query": "Where can I download past invoices?",
      "tags": [
        "keyword"
      ],
      "expected_files": [
        "invoices-and-payments.md"
      ],
      "expected_text": "PDF",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 5.479479,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 3.3870986,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 3,
          "score": 3.3651319,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 4,
          "score": 3.1572304,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 2.869647,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 2.8024464,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 7,
          "score": 2.6480045,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 2.558901,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 9,
          "score": 2.514241,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 10,
          "score": 2.4659653,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 2.4333763,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 12,
          "score": 2.4234314,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 13,
          "score": 2.3397257,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 14,
          "score": 2.3184114,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.2858236,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 2.261516,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 17,
          "score": 2.2397711,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 18,
          "score": 1.9229008,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 19,
          "score": 1.9125963,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 20,
          "score": 1.8740361,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "invoices-and-payments.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "annual-refund",
      "query": "Can I get my money back on an annual subscription?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "invoices-and-payments.md"
      ],
      "expected_text": "30 days of purchase",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 4.030309,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 2,
          "score": 3.4916003,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 3,
          "score": 3.1594803,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 3.083531,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 2.862523,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 2.7023883,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 7,
          "score": 2.6665974,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 8,
          "score": 2.5352874,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 9,
          "score": 2.4236565,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.3765519,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 11,
          "score": 2.3002856,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 12,
          "score": 2.2416017,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 2.06355,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 14,
          "score": 1.9668009,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 15,
          "score": 1.899621,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 1.8948866,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 17,
          "score": 1.8911364,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 1.875198,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 19,
          "score": 1.7308294,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "invoices-and-payments.md": 3
      },
      "snippet_found": true,
      "passage_rank": 39,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.02564102564102564
      }
    },
    {
      "id": "pay-by-wire",
      "query": "Can we pay by wire instead of by card?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "invoices-and-payments.md"
      ],
      "expected_text": "bank transfer",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 5.018153,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 3.2159462,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 3.201638,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 4,
          "score": 3.147363,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 5,
          "score": 2.6815596,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 6,
          "score": 2.6559532,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 7,
          "score": 2.5634036,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 8,
          "score": 2.364353,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 9,
          "score": 2.3472905,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 10,
          "score": 2.2389264,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 2.2166698,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 12,
          "score": 2.180029,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 2.1364298,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 14,
          "score": 2.1318357,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 15,
          "score": 2.0653362,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 16,
          "score": 1.992836,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 17,
          "score": 1.948524,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 18,
          "score": 1.9307742,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 19,
          "score": 1.7839562,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "invoices-and-payments.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "payload-retention-scale",
      "query": "How long are event bodies stored on the Scale plan?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "data-retention.md"
      ],
      "expected_text": "These are maximums",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "data-retention.md",
          "rank": 1,
          "score": 7.134757,
          "snippet_found": true
        },
        {
          "filename": "billing-plans.md",
          "rank": 2,
          "score": 5.646868,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 3,
          "score": 5.632083,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 4,
          "score": 5.5159082,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 5,
          "score": 5.322894,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 6,
          "score": 4.917618,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 4.646406,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 8,
          "score": 4.593455,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 9,
          "score": 4.1423883,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 10,
          "score": 4.102504,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 11,
          "score": 3.5780063,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 3.4474115,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 3.1836896,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 3.1513236,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 15,
          "score": 3.037868,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 16,
          "score": 2.9461775,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.7285311,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 18,
          "score": 2.6023915,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "data-retention.md": 1
      },
      "snippet_found": true,
      "passage_rank": 14,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.07142857142857142
      }
    },
    {
      "id": "erasure-request",
      "query": "A customer asked us to erase their personal data, how do I remove their events?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "data-retention.md"
      ],
      "expected_text": "Delete events",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "data-retention.md",
          "rank": 1,
          "score": 7.6901236,
          "snippet_found": true
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 2,
          "score": 6.390702,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 3,
          "score": 5.357597,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 4.2579355,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 5,
          "score": 4.2515087,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 6,
          "score": 4.1859593,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 4.102975,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 8,
          "score": 3.948564,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 9,
          "score": 3.9145777,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 10,
          "score": 3.8292284,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 3.7746987,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 12,
          "score": 3.7034235,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 13,
          "score": 3.6457257,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 3.4510388,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 3.381026,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 16,
          "score": 3.1673608,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 17,
          "score": 3.1267197,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 18,
          "score": 2.711932,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "data-retention.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "never-store-field",
      "query": "How do I keep a sensitive field from ever being saved?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "data-retention.md",
        "configuration-reference.md"
      ],
      "expected_text": "retention.redact_fields",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "transformations.md",
          "rank": 1,
          "score": 4.269417,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 2,
          "score": 4.132926,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 4.115541,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 4,
          "score": 3.9203172,
          "snippet_found": true
        },
        {
          "filename": "single-sign-on.md",
          "rank": 5,
          "score": 3.7807415,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 6,
          "score": 3.7652113,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 3.6132348,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.4858923,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 9,
          "score": 3.1855412,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 10,
          "score": 3.0955997,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 11,
          "score": 3.075557,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 12,
          "score": 2.8341694,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 13,
          "score": 2.6905017,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 14,
          "score": 2.600214,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.5657752,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 2.553431,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 17,
          "score": 2.3961258,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 18,
          "score": 2.3078952,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 2.2757673,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 20,
          "score": 2.2442474,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "data-retention.md": 4,
        "configuration-reference.md": 17
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.5,
        "recall@10": 0.5,
        "reciprocal_rank": 0.25,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.2640681225725909,
        "ndcg@10": 0.2640681225725909,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "export-to-own-storage",
      "query": "Can I keep events for longer than the plan allows by copying them to storage we own?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "data-retention.md"
      ],
      "expected_text": "scheduled export",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "data-retention.md",
          "rank": 1,
          "score": 7.004419,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 5.72364,
          "snippet_found": true
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 5.051481,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 4,
          "score": 4.8311186,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 4.7930336,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 6,
          "score": 4.558029,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 7,
          "score": 4.422286,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 8,
          "score": 4.2385845,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 4.049222,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 10,
          "score": 4.0140843,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 3.8202207,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 12,
          "score": 3.7686496,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 13,
          "score": 3.7210777,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 14,
          "score": 3.7047937,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 3.7034209,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 16,
          "score": 3.6294847,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 17,
          "score": 3.6183372,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 3.476426,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 19,
          "score": 3.2785215,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 20,
          "score": 3.2107797,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "data-retention.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "replay-event-gone",
      "query": "Replay fails saying the event isn't available any more, why?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "data-retention.md",
        "troubleshooting.md"
      ],
      "expected_text": null,
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.111964,
          "snippet_found": null
        },
        {
          "filename": "delivery-retries.md",
          "rank": 2,
          "score": 4.6264553,
          "snippet_found": null
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 4.382534,
          "snippet_found": null
        },
        {
          "filename": "error-codes.md",
          "rank": 4,
          "score": 4.204933,
          "snippet_found": null
        },
        {
          "filename": "event-filtering.md",
          "rank": 5,
          "score": 4.008878,
          "snippet_found": null
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 4.000078,
          "snippet_found": null
        },
        {
          "filename": "signing-secrets.md",
          "rank": 7,
          "score": 3.8352492,
          "snippet_found": null
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 3.6457763,
          "snippet_found": null
        },
        {
          "filename": "data-retention.md",
          "rank": 9,
          "score": 3.5607314,
          "snippet_found": null
        },
        {
          "filename": "billing-plans.md",
          "rank": 10,
          "score": 3.5086422,
          "snippet_found": null
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 11,
          "score": 3.5036902,
          "snippet_found": null
        },
        {
          "filename": "configuration-reference.md",
          "rank": 12,
          "score": 3.4338822,
          "snippet_found": null
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 3.0842357,
          "snippet_found": null
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 3.065458,
          "snippet_found": null
        },
        {
          "filename": "network-security.md",
          "rank": 15,
          "score": 2.9612713,
          "snippet_found": null
        },
        {
          "filename": "getting-started.md",
          "rank": 16,
          "score": 2.9603195,
          "snippet_found": null
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 17,
          "score": 2.8865094,
          "snippet_found": null
        },
        {
          "filename": "api-keys.md",
          "rank": 18,
          "score": 2.8189383,
          "snippet_found": null
        }
      ],
      "expected_ranks": {
        "data-retention.md": 9,
        "troubleshooting.md": 1
      },
      "snippet_found": null,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 0.5,
        "recall@5": 0.5,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 0.6131471927654584,
        "ndcg@5": 0.6131471927654584,
        "ndcg@10": 0.7977228895450266
      }
    },
    {
      "id": "eu-residency",
      "query": "We need our data to stay inside the EU, is that possible?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "regions.md"
      ],
      "expected_text": "eu-central",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "regions.md",
          "rank": 1,
          "score": 5.606211,
          "snippet_found": true
        },
        {
          "filename": "api-keys.md",
          "rank": 2,
          "score": 3.7901576,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 3,
          "score": 3.3422961,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 4,
          "score": 3.0132427,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 5,
          "score": 2.9385717,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 6,
          "score": 2.8419957,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 7,
          "score": 2.7693527,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 8,
          "score": 2.7328167,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 9,
          "score": 2.4419742,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 2.4058669,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 11,
          "score": 2.3926773,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 12,
          "score": 2.3721557,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 13,
          "score": 2.3270187,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 2.2621386,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 2.2538724,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 16,
          "score": 2.1800587,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 17,
          "score": 2.1209936,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 18,
          "score": 2.111018,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 19,
          "score": 1.9036365,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "regions.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "move-region",
      "query": "Can I move an existing workspace to another region?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "regions.md"
      ],
      "expected_text": "cannot be changed",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "regions.md",
          "rank": 1,
          "score": 7.4162574,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.9218082,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 4.279954,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 4,
          "score": 3.787121,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 5,
          "score": 3.5782912,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 6,
          "score": 3.4162853,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 3.3293962,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 8,
          "score": 3.0762732,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 3.0027943,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.9621854,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 11,
          "score": 2.8567605,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 2.7653418,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 13,
          "score": 2.742486,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 14,
          "score": 2.7398138,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.7382836,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 16,
          "score": 2.6832218,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 17,
          "score": 2.652599,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 18,
          "score": 2.6192703,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 19,
          "score": 2.2097597,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 20,
          "score": 2.1313481,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "regions.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "firewall-ips",
      "query": "Which IP addresses do I need to let through our firewall so deliveries arrive?",
      "tags": [
        "semantic",
        "multi-doc"
      ],
      "expected_files": [
        "regions.md",
        "network-security.md"
      ],
      "expected_text": "rwctl network egress-ips",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "regions.md",
          "rank": 1,
          "score": 7.6239576,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 7.329286,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 4.260872,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 4,
          "score": 3.736021,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 5,
          "score": 3.5849426,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 6,
          "score": 3.4546218,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 7,
          "score": 3.4361572,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 8,
          "score": 3.4263103,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 9,
          "score": 3.3875096,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 10,
          "score": 3.3755388,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 11,
          "score": 3.2378645,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 12,
          "score": 3.1179535,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 13,
          "score": 3.0980186,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 14,
          "score": 3.062451,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 15,
          "score": 3.0315938,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 16,
          "score": 2.801557,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 2.7423558,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 18,
          "score": 2.728594,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 19,
          "score": 2.5501351,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 20,
          "score": 2.5037901,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 21,
          "score": 2.3320217,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "regions.md": 1,
        "network-security.md": 2
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "alert-on-failing-endpoint",
      "query": "How do I get notified when an endpoint starts failing?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "monitoring-and-alerts.md"
      ],
      "expected_text": "delivery.success_rate",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 1,
          "score": 4.7551217,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 2,
          "score": 4.4837484,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 3.97679,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 4,
          "score": 3.7504718,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 3.7289722,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 6,
          "score": 3.680981,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 3.4579203,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 3.360943,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 9,
          "score": 3.2502306,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 10,
          "score": 3.0607262,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 11,
          "score": 3.031942,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 12,
          "score": 2.939257,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 13,
          "score": 2.9016738,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 14,
          "score": 2.7974367,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 15,
          "score": 2.660655,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.4786308,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.3564715,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 2.2467048,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "monitoring-and-alerts.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "metric-dlq-depth",
      "query": "dlq.depth",
      "tags": [
        "exact-token",
        "multi-doc"
      ],
      "expected_files": [
        "monitoring-and-alerts.md",
        "dead-letter-queue.md"
      ],
      "expected_text": "dlq.depth",
      "chunks_returned": 8,
      "unique_files_returned": 2,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 3.1213505,
          "snippet_found": true
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 2,
          "score": 2.994694,
          "snippet_found": true
        }
      ],
      "expected_ranks": {
        "monitoring-and-alerts.md": 2,
        "dead-letter-queue.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "alert-noise",
      "query": "How do I stop an alert from notifying us over and over?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "monitoring-and-alerts.md"
      ],
      "expected_text": "cooldown",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 1,
          "score": 5.3605433,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 2,
          "score": 4.4922824,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 3,
          "score": 3.8867335,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 4,
          "score": 3.680799,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 3.500159,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 3.3970802,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 7,
          "score": 3.2065027,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 8,
          "score": 3.1697018,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 3.005413,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 10,
          "score": 2.8459678,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 11,
          "score": 2.7075675,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 12,
          "score": 2.6995163,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 2.6899762,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 2.6840582,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 15,
          "score": 2.664556,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.4973686,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.4592614,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 18,
          "score": 2.4189043,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 19,
          "score": 2.3941422,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 20,
          "score": 2.3926606,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 21,
          "score": 2.2645342,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "monitoring-and-alerts.md": 1
      },
      "snippet_found": true,
      "passage_rank": 17,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.058823529411764705
      }
    },
    {
      "id": "metrics-history",
      "query": "How far back do the charts go?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "monitoring-and-alerts.md"
      ],
      "expected_text": "13 months",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 1,
          "score": 3.7359376,
          "snippet_found": true
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 2,
          "score": 3.1819804,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 3,
          "score": 2.9821012,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 4,
          "score": 2.8422427,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 5,
          "score": 2.8005157,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 6,
          "score": 2.783891,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 2.782668,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 8,
          "score": 2.712003,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 9,
          "score": 2.498731,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 10,
          "score": 2.2379708,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 11,
          "score": 2.218436,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 12,
          "score": 2.206134,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 2.102532,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 2.0636818,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 15,
          "score": 2.0426698,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 16,
          "score": 2.0245354,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 17,
          "score": 1.9765356,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 1.8800985,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 19,
          "score": 1.7948415,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 20,
          "score": 1.7783153,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "monitoring-and-alerts.md": 1
      },
      "snippet_found": true,
      "passage_rank": 14,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.07142857142857142
      }
    },
    {
      "id": "audit-who-changed",
      "query": "How can I find out who changed a destination's URL?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "audit-log.md"
      ],
      "expected_text": "destination.url_changed",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "invoices-and-payments.md",
          "rank": 1,
          "score": 4.8638945,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 4.4864283,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 3,
          "score": 4.4840837,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 4.211418,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 5,
          "score": 4.071476,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 6,
          "score": 4.007015,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 3.974052,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 3.8140574,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 9,
          "score": 3.7421665,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 10,
          "score": 3.5506918,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 11,
          "score": 3.4370897,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 12,
          "score": 3.4214826,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 13,
          "score": 3.28876,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 3.161325,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 3.1208224,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 16,
          "score": 2.7832265,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 2.6834779,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 18,
          "score": 2.5932093,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 19,
          "score": 2.4568262,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "audit-log.md": 3
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "audit-retention",
      "query": "How long is the record of admin actions kept?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "audit-log.md"
      ],
      "expected_text": "7 years",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "audit-log.md",
          "rank": 1,
          "score": 6.7694893,
          "snippet_found": true
        },
        {
          "filename": "data-retention.md",
          "rank": 2,
          "score": 5.902327,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 3,
          "score": 4.3268433,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 4,
          "score": 4.1397276,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 4.104513,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 6,
          "score": 4.062165,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 4.053916,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 3.2497015,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 3.210817,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 10,
          "score": 3.150314,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 11,
          "score": 3.0505908,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 12,
          "score": 2.9865234,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 13,
          "score": 2.8661394,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 2.650631,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.542187,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 16,
          "score": 2.2279477,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 2.0776277,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 18,
          "score": 1.9082611,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 19,
          "score": 1.9064578,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "audit-log.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "audit-tamper",
      "query": "Could an admin cover their tracks by deleting log entries?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "audit-log.md"
      ],
      "expected_text": "hash",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "roles-and-permissions.md",
          "rank": 1,
          "score": 5.3748984,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 2,
          "score": 5.285913,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 3,
          "score": 5.285846,
          "snippet_found": true
        },
        {
          "filename": "delivery-retries.md",
          "rank": 4,
          "score": 4.522437,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.9469934,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 6,
          "score": 3.3514473,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 3.1368482,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 8,
          "score": 3.126635,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 9,
          "score": 2.75564,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 10,
          "score": 2.7482202,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 11,
          "score": 2.715976,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 12,
          "score": 2.5803761,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 13,
          "score": 2.460242,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 14,
          "score": 2.3662038,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.3606703,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 2.197319,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 17,
          "score": 2.1412895,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 18,
          "score": 2.0583878,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "audit-log.md": 3
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "support-access",
      "query": "Can Relaywise staff look at our payloads?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "audit-log.md"
      ],
      "expected_text": "Support access",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "audit-log.md",
          "rank": 1,
          "score": 4.0565033,
          "snippet_found": true
        },
        {
          "filename": "transformations.md",
          "rank": 2,
          "score": 3.8535635,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 3,
          "score": 3.7648094,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 4,
          "score": 3.761508,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.7174695,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 6,
          "score": 3.6284103,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 7,
          "score": 3.3209238,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 3.0343482,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 9,
          "score": 2.9823442,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 10,
          "score": 2.81755,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 2.5708063,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 2.5682056,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 13,
          "score": 2.5657067,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 14,
          "score": 2.5156288,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 15,
          "score": 2.5056129,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 16,
          "score": 2.4120002,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 17,
          "score": 2.404485,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 18,
          "score": 2.3355746,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 19,
          "score": 2.2381802,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 20,
          "score": 2.1149168,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "audit-log.md": 1
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "local-development",
      "query": "How do I receive webhooks on my laptop while developing?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "cli-reference.md"
      ],
      "expected_text": "rwctl listen",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "cli-reference.md",
          "rank": 1,
          "score": 5.920323,
          "snippet_found": true
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 2,
          "score": 3.614477,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 3,
          "score": 3.504744,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 4,
          "score": 3.2636237,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 5,
          "score": 3.0580826,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 6,
          "score": 3.0453985,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 7,
          "score": 2.8457193,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 8,
          "score": 2.6562397,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 9,
          "score": 2.5232592,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 10,
          "score": 2.3091094,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 2.298862,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 12,
          "score": 2.2715812,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 13,
          "score": 2.1830935,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 14,
          "score": 1.984105,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 15,
          "score": 1.9079537,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 16,
          "score": 1.9055259,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 1.8890364,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 1.8846616,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 1.8736366,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "cli-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "cli-in-pipeline",
      "query": "How do I authenticate the command-line tool in a CI pipeline?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "cli-reference.md"
      ],
      "expected_text": "RELAYWISE_API_KEY",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "cli-reference.md",
          "rank": 1,
          "score": 6.744873,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 6.4332047,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 3,
          "score": 4.639228,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 4.624231,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 4.0451546,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 6,
          "score": 3.0904324,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 2.9595504,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 2.9179506,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 2.8892794,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 10,
          "score": 2.8273864,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 11,
          "score": 2.6481714,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 12,
          "score": 2.6116657,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 13,
          "score": 2.4750228,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 14,
          "score": 2.3840961,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.3526254,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 2.3453667,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 17,
          "score": 2.329196,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 2.3105922,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 19,
          "score": 2.0088358,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "cli-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 6,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.16666666666666666
      }
    },
    {
      "id": "cli-exit-code-5",
      "query": "What does exit code 5 from rwctl mean?",
      "tags": [
        "exact-token"
      ],
      "expected_files": [
        "cli-reference.md"
      ],
      "expected_text": "Rate limited",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "cli-reference.md",
          "rank": 1,
          "score": 6.5858765,
          "snippet_found": true
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 2,
          "score": 4.121351,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 3,
          "score": 3.7905457,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 4,
          "score": 3.750536,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 5,
          "score": 3.116803,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 6,
          "score": 3.0487542,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 2.8950555,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 8,
          "score": 2.7257786,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 9,
          "score": 2.7019591,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 10,
          "score": 2.624712,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 2.6233444,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 12,
          "score": 2.5043364,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 13,
          "score": 2.4628139,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 14,
          "score": 2.430686,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.3871965,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 2.359127,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 2.3341298,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 18,
          "score": 2.2257278,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 19,
          "score": 2.1425583,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 20,
          "score": 2.125866,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 21,
          "score": 2.0921934,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "cli-reference.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "client-certificates",
      "query": "Our endpoint requires callers to present a client certificate, is that supported?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "network-security.md"
      ],
      "expected_text": "mutual TLS",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "network-security.md",
          "rank": 1,
          "score": 9.485439,
          "snippet_found": true
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.8805037,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 3,
          "score": 4.1398563,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 4,
          "score": 3.9929626,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 5,
          "score": 3.6816976,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 6,
          "score": 3.5573359,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 7,
          "score": 3.1600873,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 3.1363287,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 9,
          "score": 3.1233253,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 3.0414858,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 11,
          "score": 2.9609323,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 2.8957937,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 2.7488675,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 14,
          "score": 2.703147,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 15,
          "score": 2.6985838,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 16,
          "score": 2.6221676,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 17,
          "score": 2.2819054,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 18,
          "score": 2.1841009,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "network-security.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "restrict-senders",
      "query": "Can I restrict which servers are allowed to send events to us?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "network-security.md"
      ],
      "expected_text": "ingest allowlist",
      "chunks_returned": 50,
      "unique_files_returned": 17,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 4.716089,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.5632486,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 4.4465055,
          "snippet_found": true
        },
        {
          "filename": "rate-limits.md",
          "rank": 4,
          "score": 4.4074636,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 5,
          "score": 4.3561697,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 6,
          "score": 4.2924013,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 7,
          "score": 4.2620754,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 8,
          "score": 4.0919485,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 9,
          "score": 4.0235558,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 10,
          "score": 3.9080744,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 3.8798714,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 3.6969318,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 13,
          "score": 3.6393707,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 14,
          "score": 3.5710607,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 15,
          "score": 3.4468415,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 3.0487945,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 17,
          "score": 2.857802,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "network-security.md": 3
      },
      "snippet_found": true,
      "passage_rank": 3,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.3333333333333333
      }
    },
    {
      "id": "enforce-two-step",
      "query": "How do I force everyone to use two-step login?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "network-security.md"
      ],
      "expected_text": "Require two-factor",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "single-sign-on.md",
          "rank": 1,
          "score": 4.196766,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.1229253,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 3.2927356,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 4,
          "score": 3.2802827,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 5,
          "score": 3.2128882,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 6,
          "score": 3.0439441,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 7,
          "score": 2.7201033,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 8,
          "score": 2.644678,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 9,
          "score": 2.5467582,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 10,
          "score": 2.535389,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 11,
          "score": 2.4999475,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 12,
          "score": 2.4157667,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 13,
          "score": 2.4001741,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 14,
          "score": 2.3948617,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 15,
          "score": 2.3878696,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 16,
          "score": 2.282506,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.2025251,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 2.0695238,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 19,
          "score": 1.9594917,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 20,
          "score": 1.9291121,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 21,
          "score": 1.796094,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "network-security.md": 3
      },
      "snippet_found": true,
      "passage_rank": 9,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.3333333333333333,
        "ndcg@1": 0.0,
        "ndcg@3": 0.5,
        "ndcg@5": 0.5,
        "ndcg@10": 0.5,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.1111111111111111
      }
    },
    {
      "id": "encryption-at-rest",
      "query": "Is stored data encrypted?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "network-security.md"
      ],
      "expected_text": "AES-256",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "network-security.md",
          "rank": 1,
          "score": 4.051931,
          "snippet_found": true
        },
        {
          "filename": "data-retention.md",
          "rank": 2,
          "score": 3.8924236,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 3,
          "score": 3.4245567,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 4,
          "score": 2.852664,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 2.5582569,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 6,
          "score": 2.4678826,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 7,
          "score": 2.4120307,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 8,
          "score": 2.2670894,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 9,
          "score": 2.194287,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 10,
          "score": 2.1178303,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 11,
          "score": 1.8928216,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 12,
          "score": 1.864538,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 13,
          "score": 1.8635015,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 14,
          "score": 1.7791486,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 15,
          "score": 1.5821353,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 16,
          "score": 1.5647495,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 17,
          "score": 1.5531394,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 1.5409894,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 19,
          "score": 1.4597204,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "network-security.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "nothing-arriving",
      "query": "My endpoint isn't getting any events, where do I start looking?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "troubleshooting.md"
      ],
      "expected_text": "Was the event accepted",
      "chunks_returned": 50,
      "unique_files_returned": 19,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.6265793,
          "snippet_found": true
        },
        {
          "filename": "getting-started.md",
          "rank": 2,
          "score": 4.9406214,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 3,
          "score": 4.242301,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 4,
          "score": 4.122987,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 5,
          "score": 4.0675335,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 6,
          "score": 3.9629734,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 7,
          "score": 3.9350436,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 8,
          "score": 3.9290373,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 9,
          "score": 3.8895578,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 10,
          "score": 3.573028,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 11,
          "score": 3.5166721,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 12,
          "score": 3.334948,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 13,
          "score": 3.2872982,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 3.243579,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 15,
          "score": 3.0642722,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 16,
          "score": 3.0577633,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 17,
          "score": 3.028301,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 2.8070688,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 19,
          "score": 2.774746,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "troubleshooting.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "lost-authenticator",
      "query": "I lost my authenticator app, how do I get back in?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "troubleshooting.md"
      ],
      "expected_text": "second factor",
      "chunks_returned": 50,
      "unique_files_returned": 21,
      "ranked_files": [
        {
          "filename": "troubleshooting.md",
          "rank": 1,
          "score": 5.154069,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 2,
          "score": 4.039789,
          "snippet_found": true
        },
        {
          "filename": "single-sign-on.md",
          "rank": 3,
          "score": 3.4582567,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 4,
          "score": 3.15412,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 5,
          "score": 3.0289927,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 3.0060077,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 7,
          "score": 2.9968257,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 8,
          "score": 2.9520168,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 9,
          "score": 2.8144894,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 10,
          "score": 2.6343243,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 11,
          "score": 2.5093923,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 12,
          "score": 2.4954455,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 2.4666529,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 14,
          "score": 2.3090854,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 15,
          "score": 2.2876267,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 16,
          "score": 2.262417,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 17,
          "score": 2.1686492,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 18,
          "score": 2.117509,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 19,
          "score": 2.0579886,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 20,
          "score": 1.9386961,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 21,
          "score": 1.9223454,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "troubleshooting.md": 1
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 1.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    },
    {
      "id": "first-test-event",
      "query": "What is the quickest way to send a first test event?",
      "tags": [
        "semantic"
      ],
      "expected_files": [
        "getting-started.md"
      ],
      "expected_text": "202 Accepted",
      "chunks_returned": 50,
      "unique_files_returned": 18,
      "ranked_files": [
        {
          "filename": "dead-letter-queue.md",
          "rank": 1,
          "score": 5.160555,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 2,
          "score": 4.864584,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 3,
          "score": 4.5930367,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 4,
          "score": 4.4245358,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 5,
          "score": 4.3252854,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 6,
          "score": 4.2423105,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 7,
          "score": 4.239122,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 8,
          "score": 4.103107,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 9,
          "score": 3.8884325,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 10,
          "score": 3.7158773,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 11,
          "score": 3.655969,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 12,
          "score": 3.6182408,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 13,
          "score": 3.407936,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 14,
          "score": 3.300753,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 15,
          "score": 3.2260537,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 16,
          "score": 3.218966,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 3.0701206,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 18,
          "score": 3.029202,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "getting-started.md": 7
      },
      "snippet_found": false,
      "passage_rank": null,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 0.0,
        "recall@5": 0.0,
        "recall@10": 0.0,
        "reciprocal_rank": 0.0,
        "ndcg@1": 0.0,
        "ndcg@3": 0.0,
        "ndcg@5": 0.0,
        "ndcg@10": 0.0,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 0.0,
        "passage_hit@10": 0.0,
        "passage_rr": 0.0
      }
    },
    {
      "id": "source-vs-destination",
      "query": "What is the difference between a source and a destination?",
      "tags": [
        "keyword"
      ],
      "expected_files": [
        "getting-started.md"
      ],
      "expected_text": "Where events come from",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "cli-reference.md",
          "rank": 1,
          "score": 4.9485564,
          "snippet_found": false
        },
        {
          "filename": "getting-started.md",
          "rank": 2,
          "score": 4.9001284,
          "snippet_found": true
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 3,
          "score": 3.985413,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 4,
          "score": 3.8785093,
          "snippet_found": false
        },
        {
          "filename": "network-security.md",
          "rank": 5,
          "score": 3.8600652,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 6,
          "score": 3.7427316,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 7,
          "score": 3.673921,
          "snippet_found": false
        },
        {
          "filename": "error-codes.md",
          "rank": 8,
          "score": 3.6201513,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 9,
          "score": 3.4754732,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 10,
          "score": 3.3986077,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 11,
          "score": 3.2834342,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 12,
          "score": 3.2481108,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 13,
          "score": 3.2231283,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 14,
          "score": 3.2037265,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 15,
          "score": 3.149214,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 16,
          "score": 3.0159397,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 17,
          "score": 2.9234078,
          "snippet_found": false
        },
        {
          "filename": "data-retention.md",
          "rank": 18,
          "score": 2.8746972,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 19,
          "score": 2.7781506,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 20,
          "score": 2.455618,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "getting-started.md": 2
      },
      "snippet_found": true,
      "passage_rank": 4,
      "metrics": {
        "recall@1": 0.0,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 0.5,
        "ndcg@1": 0.0,
        "ndcg@3": 0.6309297535714575,
        "ndcg@5": 0.6309297535714575,
        "ndcg@10": 0.6309297535714575,
        "passage_hit@1": 0.0,
        "passage_hit@3": 0.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 0.25
      }
    },
    {
      "id": "plain-http-destination",
      "query": "Can a destination use a plain http URL?",
      "tags": [
        "keyword",
        "multi-doc"
      ],
      "expected_files": [
        "getting-started.md",
        "error-codes.md"
      ],
      "expected_text": "localhost",
      "chunks_returned": 50,
      "unique_files_returned": 20,
      "ranked_files": [
        {
          "filename": "getting-started.md",
          "rank": 1,
          "score": 6.999161,
          "snippet_found": true
        },
        {
          "filename": "error-codes.md",
          "rank": 2,
          "score": 6.419933,
          "snippet_found": true
        },
        {
          "filename": "network-security.md",
          "rank": 3,
          "score": 3.9198084,
          "snippet_found": false
        },
        {
          "filename": "cli-reference.md",
          "rank": 4,
          "score": 3.8565774,
          "snippet_found": false
        },
        {
          "filename": "dead-letter-queue.md",
          "rank": 5,
          "score": 3.5010269,
          "snippet_found": false
        },
        {
          "filename": "delivery-retries.md",
          "rank": 6,
          "score": 3.4314427,
          "snippet_found": false
        },
        {
          "filename": "event-filtering.md",
          "rank": 7,
          "score": 3.1986198,
          "snippet_found": false
        },
        {
          "filename": "transformations.md",
          "rank": 8,
          "score": 3.1823845,
          "snippet_found": false
        },
        {
          "filename": "signing-secrets.md",
          "rank": 9,
          "score": 3.1128054,
          "snippet_found": false
        },
        {
          "filename": "audit-log.md",
          "rank": 10,
          "score": 3.0919387,
          "snippet_found": false
        },
        {
          "filename": "configuration-reference.md",
          "rank": 11,
          "score": 2.918428,
          "snippet_found": false
        },
        {
          "filename": "monitoring-and-alerts.md",
          "rank": 12,
          "score": 2.7586346,
          "snippet_found": false
        },
        {
          "filename": "troubleshooting.md",
          "rank": 13,
          "score": 2.7350254,
          "snippet_found": false
        },
        {
          "filename": "regions.md",
          "rank": 14,
          "score": 2.7063363,
          "snippet_found": false
        },
        {
          "filename": "single-sign-on.md",
          "rank": 15,
          "score": 2.550699,
          "snippet_found": false
        },
        {
          "filename": "rate-limits.md",
          "rank": 16,
          "score": 2.3362465,
          "snippet_found": false
        },
        {
          "filename": "invoices-and-payments.md",
          "rank": 17,
          "score": 2.2635229,
          "snippet_found": false
        },
        {
          "filename": "roles-and-permissions.md",
          "rank": 18,
          "score": 2.2386634,
          "snippet_found": false
        },
        {
          "filename": "billing-plans.md",
          "rank": 19,
          "score": 2.2386358,
          "snippet_found": false
        },
        {
          "filename": "api-keys.md",
          "rank": 20,
          "score": 2.1976867,
          "snippet_found": false
        }
      ],
      "expected_ranks": {
        "getting-started.md": 1,
        "error-codes.md": 2
      },
      "snippet_found": true,
      "passage_rank": 1,
      "metrics": {
        "recall@1": 0.5,
        "recall@3": 1.0,
        "recall@5": 1.0,
        "recall@10": 1.0,
        "reciprocal_rank": 1.0,
        "ndcg@1": 1.0,
        "ndcg@3": 1.0,
        "ndcg@5": 1.0,
        "ndcg@10": 1.0,
        "passage_hit@1": 1.0,
        "passage_hit@3": 1.0,
        "passage_hit@5": 1.0,
        "passage_hit@10": 1.0,
        "passage_rr": 1.0
      }
    }
  ]
}