OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
{
"schema_version": 1,
"run": {
"run_id": "eval-49debd2f",
"timestamp": "2026-11-05T18:03:57Z",
"duration_s": 85.9,
"target_url": "http://localhost:3000",
"sdk_version": "0.4.0.dev0",
"git_commit": "a7076dc551c4b940b63119b9c242665fa927f880",
"corpus_files": 21,
"corpus_hash": "9a346ec3274599771b53573f708fca382c23a27b19d102a84210377df06ab73a",
"query_count": 83,
"passage_questions": 77,
"queries_hash": "a76d3448549d17810642d5a5206a56f54b8e8f605b43def0d3791941ba9f31bf",
"k_values": [
1,
3,
5,
10
],
"primary_k": 5,
"fetch_limit": 50,
"score_threshold": 0.0
},
"settings": {
"embedding_provider": "azure",
"embedding_model": "text-embedding-3-small",
"chunk_size": 1000,
"chunk_overlap": 200,
"table_structure": true,
"ocr": false,
"picture_descriptions": false
},
"settings_note": null,
"metrics": {
"overall": {
"recall@1": 0.6265060240963856,
"recall@3": 0.8493975903614458,
"recall@5": 0.9096385542168675,
"recall@10": 0.963855421686747,
"mrr": 0.7938898450946643,
"ndcg@1": 0.7108433734939759,
"ndcg@3": 0.7762517657398734,
"ndcg@5": 0.8030453462731483,
"ndcg@10": 0.8211616182694974,
"passage_hit@1": 0.5324675324675324,
"passage_hit@3": 0.7012987012987013,
"passage_hit@5": 0.7922077922077922,
"passage_hit@10": 0.8701298701298701,
"passage_mrr": 0.6384012790334507
},
"by_tag": {
"exact-token": {
"count": 13,
"passage_count": 13,
"recall@1": 0.7307692307692307,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"mrr": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.9876493521766443,
"ndcg@5": 0.9876493521766443,
"ndcg@10": 0.9876493521766443,
"passage_hit@1": 0.8461538461538461,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_mrr": 0.9230769230769231
},
"keyword": {
"count": 7,
"passage_count": 6,
"recall@1": 0.5714285714285714,
"recall@3": 0.9285714285714286,
"recall@5": 0.9285714285714286,
"recall@10": 1.0,
"mrr": 0.8333333333333333,
"ndcg@1": 0.7142857142857143,
"ndcg@3": 0.8205824209052738,
"ndcg@5": 0.8205824209052738,
"ndcg@10": 0.8459023190172805,
"passage_hit@1": 0.6666666666666666,
"passage_hit@3": 0.8333333333333334,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_mrr": 0.7638888888888888
},
"multi-doc": {
"count": 16,
"passage_count": 11,
"recall@1": 0.4375,
"recall@3": 0.71875,
"recall@5": 0.84375,
"recall@10": 0.9375,
"mrr": 0.903125,
"ndcg@1": 0.875,
"ndcg@3": 0.7440735963827292,
"ndcg@5": 0.8050524251877592,
"ndcg@10": 0.8413163264616205,
"passage_hit@1": 0.7272727272727273,
"passage_hit@3": 0.9090909090909091,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_mrr": 0.8257575757575758
},
"semantic": {
"count": 63,
"passage_count": 58,
"recall@1": 0.6111111111111112,
"recall@3": 0.8095238095238095,
"recall@5": 0.8888888888888888,
"recall@10": 0.9523809523809523,
"mrr": 0.7469765684051398,
"ndcg@1": 0.6507936507936508,
"ndcg@3": 0.7277044132027969,
"ndcg@5": 0.7630038923180636,
"ndcg@10": 0.7840580397611262,
"passage_hit@1": 0.4482758620689655,
"passage_hit@3": 0.6206896551724138,
"passage_hit@5": 0.7241379310344828,
"passage_hit@10": 0.8275862068965517,
"passage_mrr": 0.5616131922800408
}
}
},
"queries": [
{
"id": "key-rotate",
"query": "How do I swap out an API key without causing downtime?",
"tags": [
"semantic"
],
"expected_files": [
"api-keys.md"
],
"expected_text": "overlap window",
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "api-keys.md",
"rank": 1,
"score": 4.485182,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.1399813,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 3.9088736,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 4,
"score": 3.9022071,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.8871171,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 6,
"score": 3.7825556,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 7,
"score": 3.5549948,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 3.3526278,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 3.1453872,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 3.0133064,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 2.9531522,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 12,
"score": 2.9356976,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 2.8147604,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 2.790321,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 2.6110516,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 16,
"score": 2.5505896,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.3984993,
"snippet_found": false
}
],
"expected_ranks": {
"api-keys.md": 1
},
"snippet_found": true,
"passage_rank": 10,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.1
}
},
{
"id": "key-leaked",
"query": "One of our keys was pushed to a public repo, what should we do right now?",
"tags": [
"semantic"
],
"expected_files": [
"api-keys.md"
],
"expected_text": "Revoke immediately",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "api-keys.md",
"rank": 1,
"score": 4.5577087,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 2,
"score": 4.4239907,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 3.8773596,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 4,
"score": 3.7586002,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 5,
"score": 3.7468228,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 6,
"score": 3.6312208,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 7,
"score": 3.373435,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 8,
"score": 3.268697,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 9,
"score": 3.2684705,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 3.224593,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 3.1721559,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 3.1261327,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 13,
"score": 3.0540926,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 14,
"score": 3.023003,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.9438276,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 16,
"score": 2.9110684,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 2.8642788,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 2.65653,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 19,
"score": 2.5132947,
"snippet_found": false
}
],
"expected_ranks": {
"api-keys.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "key-scopes",
"query": "What scopes can I give an API key?",
"tags": [
"keyword"
],
"expected_files": [
"api-keys.md"
],
"expected_text": "destinations:manage",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "api-keys.md",
"rank": 1,
"score": 5.5974355,
"snippet_found": true
},
{
"filename": "error-codes.md",
"rank": 2,
"score": 5.0317492,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 3,
"score": 4.3466816,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 3.7902362,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.6450343,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 6,
"score": 3.5535414,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 7,
"score": 3.533179,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 8,
"score": 3.477205,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 9,
"score": 3.3471954,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 3.334623,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 3.179077,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 12,
"score": 3.147795,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 13,
"score": 3.0607226,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 14,
"score": 2.8921738,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.697281,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 2.5704966,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 2.479469,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 18,
"score": 2.3684938,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 19,
"score": 2.3103514,
"snippet_found": false
}
],
"expected_ranks": {
"api-keys.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "key-prefix-ci",
"query": "What is a key starting with rwk_ci_ used for?",
"tags": [
"exact-token"
],
"expected_files": [
"api-keys.md"
],
"expected_text": "Automation key",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "api-keys.md",
"rank": 1,
"score": 5.3789306,
"snippet_found": true
},
{
"filename": "cli-reference.md",
"rank": 2,
"score": 4.635419,
"snippet_found": true
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 3.814974,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 4,
"score": 3.5494313,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 5,
"score": 3.0389695,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 6,
"score": 2.91465,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 2.8971639,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 2.8961177,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 9,
"score": 2.875177,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.8743634,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 2.7834806,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 2.7593927,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 13,
"score": 2.5931182,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 14,
"score": 2.5388367,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 2.5265174,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 2.4976048,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.3593998,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 18,
"score": 2.2960348,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 19,
"score": 2.2928317,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 20,
"score": 2.171982,
"snippet_found": false
}
],
"expected_ranks": {
"api-keys.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "sig-verify",
"query": "How can my server confirm that a webhook really came from Relaywise and was not forged?",
"tags": [
"semantic"
],
"expected_files": [
"signing-secrets.md"
],
"expected_text": "HMAC-SHA256",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "signing-secrets.md",
"rank": 1,
"score": 5.4620786,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 2,
"score": 5.1428657,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 5.107025,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 4,
"score": 4.8653693,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 5,
"score": 4.459428,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 6,
"score": 4.323626,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 4.092972,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 8,
"score": 4.032093,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 9,
"score": 3.9983268,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 10,
"score": 3.9548411,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 11,
"score": 3.8796291,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 12,
"score": 3.8189056,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 3.8055673,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 14,
"score": 3.7612507,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 15,
"score": 3.748525,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 3.7323694,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 3.6424484,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 18,
"score": 3.5377412,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 19,
"score": 3.460004,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 20,
"score": 3.3449996,
"snippet_found": false
}
],
"expected_ranks": {
"signing-secrets.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "sig-roll",
"query": "How do I change the secret used to sign deliveries, and how long do the old and new ones both work?",
"tags": [
"semantic"
],
"expected_files": [
"signing-secrets.md"
],
"expected_text": "72 hours",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "audit-log.md",
"rank": 1,
"score": 8.1549225,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 2,
"score": 7.6038017,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 3,
"score": 7.401187,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 4,
"score": 6.34908,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 6.135564,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 6,
"score": 5.6227202,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 7,
"score": 5.285435,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 5.0490365,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 9,
"score": 4.9172583,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 10,
"score": 4.916431,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 11,
"score": 4.8609548,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 12,
"score": 4.823242,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 13,
"score": 4.7858067,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 14,
"score": 4.7696395,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 4.65145,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 4.4600706,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 17,
"score": 4.159475,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 18,
"score": 3.9943762,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 19,
"score": 3.8904157,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 20,
"score": 3.7599213,
"snippet_found": false
}
],
"expected_ranks": {
"signing-secrets.md": 3
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "sig-max-age",
"query": "How old can a signed request be before I should refuse it?",
"tags": [
"semantic"
],
"expected_files": [
"signing-secrets.md"
],
"expected_text": "five minutes",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "signing-secrets.md",
"rank": 1,
"score": 5.4995303,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 5.3574557,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 4.620363,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 4,
"score": 4.157878,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 5,
"score": 3.9991243,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 6,
"score": 3.7961216,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 3.7712054,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 3.6903996,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 9,
"score": 3.672656,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 3.3198228,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 11,
"score": 3.3114727,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 3.3018017,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 13,
"score": 3.2214346,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 3.219668,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 2.914604,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 2.8591857,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.8397179,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 18,
"score": 2.7246382,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 19,
"score": 2.6846876,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 20,
"score": 2.6837616,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 21,
"score": 2.5916595,
"snippet_found": false
}
],
"expected_ranks": {
"signing-secrets.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "sig-whsec",
"query": "Where do I find the whsec_ value for a destination?",
"tags": [
"exact-token"
],
"expected_files": [
"signing-secrets.md"
],
"expected_text": "Reveal",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "signing-secrets.md",
"rank": 1,
"score": 6.0598135,
"snippet_found": true
},
{
"filename": "configuration-reference.md",
"rank": 2,
"score": 4.84408,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 4.648713,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.378025,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.803798,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 6,
"score": 3.4391446,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 3.3304148,
"snippet_found": true
},
{
"filename": "regions.md",
"rank": 8,
"score": 3.3192945,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 9,
"score": 2.9882765,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 10,
"score": 2.8431144,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 11,
"score": 2.8061593,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 12,
"score": 2.6899495,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 13,
"score": 2.6079462,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 14,
"score": 2.5567536,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 2.555109,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 2.4441526,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 17,
"score": 2.355443,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 18,
"score": 2.2208116,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 19,
"score": 2.1636539,
"snippet_found": false
}
],
"expected_ranks": {
"signing-secrets.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "sso-setup",
"query": "How do we let staff log in to the console with their company accounts?",
"tags": [
"semantic"
],
"expected_files": [
"single-sign-on.md"
],
"expected_text": "identity provider",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "audit-log.md",
"rank": 1,
"score": 4.916114,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 2,
"score": 4.913085,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 4.8817015,
"snippet_found": true
},
{
"filename": "cli-reference.md",
"rank": 4,
"score": 4.7266417,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 5,
"score": 4.4032316,
"snippet_found": true
},
{
"filename": "invoices-and-payments.md",
"rank": 6,
"score": 4.382999,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 7,
"score": 4.0702477,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 8,
"score": 4.055292,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 3.9624317,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 10,
"score": 3.7526836,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 11,
"score": 3.4365666,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 12,
"score": 3.0746837,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 13,
"score": 3.007635,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 14,
"score": 2.8946323,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 2.8831072,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 2.7998972,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.716467,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 18,
"score": 2.7119622,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 19,
"score": 2.5842686,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 20,
"score": 2.5467687,
"snippet_found": false
}
],
"expected_ranks": {
"single-sign-on.md": 5
},
"snippet_found": true,
"passage_rank": 6,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.2,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.38685280723454163,
"ndcg@10": 0.38685280723454163,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.16666666666666666
}
},
{
"id": "sso-lockout",
"query": "If our identity provider goes down, can anyone still get into the console?",
"tags": [
"semantic"
],
"expected_files": [
"single-sign-on.md"
],
"expected_text": "break-glass",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "single-sign-on.md",
"rank": 1,
"score": 7.240963,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 3.965464,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 3.9560423,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 4,
"score": 3.5496902,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 5,
"score": 3.2137072,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 6,
"score": 3.2129083,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 7,
"score": 3.2106323,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.1655471,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 9,
"score": 3.0810347,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 10,
"score": 3.0189486,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 2.8896675,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 2.8698277,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 2.6514459,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 14,
"score": 2.6178923,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 2.6146202,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 2.5582738,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 17,
"score": 2.5566497,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 2.5499167,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 19,
"score": 2.1304684,
"snippet_found": false
}
],
"expected_ranks": {
"single-sign-on.md": 1
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "sso-offboarding",
"query": "How is someone's access removed automatically when they leave the company?",
"tags": [
"semantic"
],
"expected_files": [
"single-sign-on.md"
],
"expected_text": "SCIM",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "single-sign-on.md",
"rank": 1,
"score": 5.3100443,
"snippet_found": true
},
{
"filename": "signing-secrets.md",
"rank": 2,
"score": 4.564142,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 3,
"score": 4.112088,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 4,
"score": 4.080737,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 5,
"score": 3.9498796,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 6,
"score": 3.8103957,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 7,
"score": 3.748855,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 3.6031716,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 9,
"score": 3.5195658,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 3.4794078,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 11,
"score": 3.4275222,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 12,
"score": 3.338546,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 13,
"score": 3.278238,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 3.2054431,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 15,
"score": 3.070385,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 16,
"score": 2.9049387,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 17,
"score": 2.4249506,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 18,
"score": 2.311882,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 19,
"score": 2.2950606,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 20,
"score": 2.2608783,
"snippet_found": false
}
],
"expected_ranks": {
"single-sign-on.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "session-length",
"query": "How long before the console logs me out?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"single-sign-on.md",
"configuration-reference.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "network-security.md",
"rank": 1,
"score": 4.718949,
"snippet_found": null
},
{
"filename": "audit-log.md",
"rank": 2,
"score": 4.631812,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 3,
"score": 4.393179,
"snippet_found": null
},
{
"filename": "transformations.md",
"rank": 4,
"score": 3.361267,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 3.2693686,
"snippet_found": null
},
{
"filename": "single-sign-on.md",
"rank": 6,
"score": 3.2635205,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 7,
"score": 3.2346013,
"snippet_found": null
},
{
"filename": "roles-and-permissions.md",
"rank": 8,
"score": 3.1757994,
"snippet_found": null
},
{
"filename": "billing-plans.md",
"rank": 9,
"score": 3.057241,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 10,
"score": 2.9434013,
"snippet_found": null
},
{
"filename": "rate-limits.md",
"rank": 11,
"score": 2.8661394,
"snippet_found": null
},
{
"filename": "dead-letter-queue.md",
"rank": 12,
"score": 2.8531485,
"snippet_found": null
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 2.7566361,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 14,
"score": 2.6207955,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 15,
"score": 2.4442549,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 16,
"score": 2.4371514,
"snippet_found": null
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 2.3959312,
"snippet_found": null
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 2.392912,
"snippet_found": null
},
{
"filename": "api-keys.md",
"rank": 19,
"score": 2.146281,
"snippet_found": null
},
{
"filename": "monitoring-and-alerts.md",
"rank": 20,
"score": 1.9371018,
"snippet_found": null
}
],
"expected_ranks": {
"single-sign-on.md": 6,
"configuration-reference.md": 5
},
"snippet_found": null,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.5,
"recall@10": 1.0,
"reciprocal_rank": 0.2,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.23719771276929622,
"ndcg@10": 0.45560514958746035
}
},
{
"id": "dlq-purge-who",
"query": "Who is allowed to empty a dead-letter queue?",
"tags": [
"keyword",
"multi-doc"
],
"expected_files": [
"roles-and-permissions.md",
"dead-letter-queue.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 5.936977,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 2,
"score": 5.469982,
"snippet_found": null
},
{
"filename": "api-keys.md",
"rank": 3,
"score": 4.6652865,
"snippet_found": null
},
{
"filename": "troubleshooting.md",
"rank": 4,
"score": 4.4582715,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 5,
"score": 4.1814966,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 6,
"score": 4.075127,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 7,
"score": 4.028639,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 8,
"score": 3.9006016,
"snippet_found": null
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 3.8686628,
"snippet_found": null
},
{
"filename": "roles-and-permissions.md",
"rank": 10,
"score": 3.857781,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 11,
"score": 3.807433,
"snippet_found": null
},
{
"filename": "transformations.md",
"rank": 12,
"score": 3.4361827,
"snippet_found": null
},
{
"filename": "network-security.md",
"rank": 13,
"score": 2.9324346,
"snippet_found": null
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 2.883803,
"snippet_found": null
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 2.7358198,
"snippet_found": null
},
{
"filename": "rate-limits.md",
"rank": 16,
"score": 2.616968,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 17,
"score": 2.392273,
"snippet_found": null
},
{
"filename": "event-filtering.md",
"rank": 18,
"score": 2.3337693,
"snippet_found": null
}
],
"expected_ranks": {
"roles-and-permissions.md": 10,
"dead-letter-queue.md": 1
},
"snippet_found": null,
"metrics": {
"recall@1": 0.5,
"recall@3": 0.5,
"recall@5": 0.5,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.6131471927654584,
"ndcg@5": 0.6131471927654584,
"ndcg@10": 0.7903864795495061
}
},
{
"id": "viewer-payloads",
"query": "Can a read-only user see the contents of events?",
"tags": [
"semantic"
],
"expected_files": [
"roles-and-permissions.md"
],
"expected_text": "Payload access",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "roles-and-permissions.md",
"rank": 1,
"score": 4.9205885,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.8499303,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 3,
"score": 4.828339,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.6417646,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 5,
"score": 4.445083,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 6,
"score": 4.2635155,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 7,
"score": 3.988793,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 8,
"score": 3.8740609,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 3.845269,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 3.5629725,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 3.4642334,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 3.359555,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 13,
"score": 3.3335032,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 14,
"score": 3.3266916,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 3.1584954,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 16,
"score": 3.0837433,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 17,
"score": 3.0490007,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 2.998888,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 19,
"score": 2.7927902,
"snippet_found": false
}
],
"expected_ranks": {
"roles-and-permissions.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "invite-expiry",
"query": "How long does a team invitation link stay valid?",
"tags": [
"semantic"
],
"expected_files": [
"roles-and-permissions.md"
],
"expected_text": "valid for 7 days",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "roles-and-permissions.md",
"rank": 1,
"score": 5.6239557,
"snippet_found": true
},
{
"filename": "audit-log.md",
"rank": 2,
"score": 4.895487,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 3,
"score": 4.1866374,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 4.109663,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 5,
"score": 4.0150146,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 6,
"score": 3.691551,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 3.5175457,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 3.4534285,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 9,
"score": 3.2989433,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 3.2002692,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 11,
"score": 2.9563212,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 12,
"score": 2.823382,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 2.8055763,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 14,
"score": 2.7686763,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 2.7495055,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 16,
"score": 2.7170486,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 17,
"score": 2.7069454,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 2.6110837,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 19,
"score": 2.4631553,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 20,
"score": 2.0309317,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 21,
"score": 2.0022187,
"snippet_found": false
}
],
"expected_ranks": {
"roles-and-permissions.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "ownership-transfer",
"query": "How do I hand a workspace over to a colleague?",
"tags": [
"semantic"
],
"expected_files": [
"roles-and-permissions.md"
],
"expected_text": "last Owner",
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 4.235373,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 2,
"score": 4.2214937,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 3,
"score": 4.050256,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 4,
"score": 3.4798121,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 5,
"score": 3.3237166,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 3.3111908,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 7,
"score": 3.0920124,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 8,
"score": 3.0649781,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 2.9621043,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.6565669,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 11,
"score": 2.6309865,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 2.5766845,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 2.551534,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 2.524885,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 15,
"score": 2.5160358,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 16,
"score": 2.360861,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 17,
"score": 2.2255254,
"snippet_found": false
}
],
"expected_ranks": {
"roles-and-permissions.md": 4
},
"snippet_found": true,
"passage_rank": 5,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.25,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.43067655807339306,
"ndcg@10": 0.43067655807339306,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.2
}
},
{
"id": "retry-schedule",
"query": "How many times will a failed webhook be attempted, and over how long?",
"tags": [
"semantic"
],
"expected_files": [
"delivery-retries.md"
],
"expected_text": "27 hours",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "delivery-retries.md",
"rank": 1,
"score": 5.953916,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 5.9165845,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 3,
"score": 5.629542,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 5.3382273,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 5,
"score": 5.2244463,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 6,
"score": 4.7950573,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 4.6740932,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 8,
"score": 4.488102,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 4.288582,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 10,
"score": 4.247769,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 11,
"score": 4.176223,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 12,
"score": 4.0876427,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 3.7292528,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 14,
"score": 3.6605527,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 15,
"score": 3.4551523,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 16,
"score": 3.4263456,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 17,
"score": 3.2239738,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 2.5783687,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 19,
"score": 2.4560182,
"snippet_found": false
}
],
"expected_ranks": {
"delivery-retries.md": 1
},
"snippet_found": true,
"passage_rank": 6,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.16666666666666666
}
},
{
"id": "cfg-max-attempts",
"query": "What is the default for delivery.max_attempts?",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"delivery-retries.md",
"configuration-reference.md"
],
"expected_text": "delivery.max_attempts",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "delivery-retries.md",
"rank": 1,
"score": 4.100553,
"snippet_found": true
},
{
"filename": "configuration-reference.md",
"rank": 2,
"score": 3.990864,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 3,
"score": 2.4812531,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 2.3984542,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 5,
"score": 2.262816,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 6,
"score": 2.2063332,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 7,
"score": 2.1881351,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 8,
"score": 2.1569738,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 9,
"score": 2.0785315,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.0601966,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 11,
"score": 2.0537386,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 12,
"score": 2.0393662,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 13,
"score": 2.037712,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 14,
"score": 1.9448432,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 15,
"score": 1.8971537,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 1.8890164,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 17,
"score": 1.8565283,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 18,
"score": 1.8449395,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 19,
"score": 1.8378721,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 20,
"score": 1.8362129,
"snippet_found": false
}
],
"expected_ranks": {
"delivery-retries.md": 1,
"configuration-reference.md": 2
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "circuit-breaker",
"query": "What happens when most requests to one endpoint start failing?",
"tags": [
"semantic"
],
"expected_files": [
"delivery-retries.md"
],
"expected_text": "degraded",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "monitoring-and-alerts.md",
"rank": 1,
"score": 5.0543203,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 2,
"score": 4.903905,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 4.752765,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.5258617,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 5,
"score": 4.1057053,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 6,
"score": 4.084237,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 7,
"score": 3.9673247,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.947473,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 3.8320982,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 10,
"score": 3.7889988,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 11,
"score": 3.705308,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 12,
"score": 3.6559482,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 13,
"score": 3.598694,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 14,
"score": 3.2583046,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 15,
"score": 3.1484778,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 16,
"score": 2.9071856,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 2.8332481,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 2.7670798,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 19,
"score": 2.5437899,
"snippet_found": false
}
],
"expected_ranks": {
"delivery-retries.md": 2
},
"snippet_found": true,
"passage_rank": 2,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.5,
"ndcg@1": 0.0,
"ndcg@3": 0.6309297535714575,
"ndcg@5": 0.6309297535714575,
"ndcg@10": 0.6309297535714575,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.5
}
},
{
"id": "gone-410",
"query": "What does Relaywise do if my endpoint answers 410 Gone?",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"delivery-retries.md",
"dead-letter-queue.md"
],
"expected_text": "410 Gone",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "delivery-retries.md",
"rank": 1,
"score": 5.6472993,
"snippet_found": true
},
{
"filename": "error-codes.md",
"rank": 2,
"score": 4.9761944,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 4.925442,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 4,
"score": 4.794298,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 5,
"score": 4.408581,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 6,
"score": 4.156478,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 7,
"score": 4.053644,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 3.842249,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 9,
"score": 3.769952,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 10,
"score": 3.6307647,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 3.591357,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 12,
"score": 3.158509,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 13,
"score": 3.0485291,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 14,
"score": 2.950628,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.7675254,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 2.7243469,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.7232513,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 18,
"score": 2.5460482,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 19,
"score": 2.5377195,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 20,
"score": 2.4653406,
"snippet_found": false
}
],
"expected_ranks": {
"delivery-retries.md": 1,
"dead-letter-queue.md": 3
},
"snippet_found": true,
"passage_rank": 2,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.9197207891481876,
"ndcg@5": 0.9197207891481876,
"ndcg@10": 0.9197207891481876,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.5
}
},
{
"id": "duplicate-events",
"query": "Why did my endpoint get the same event twice?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"delivery-retries.md",
"troubleshooting.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 6.117975,
"snippet_found": null
},
{
"filename": "event-filtering.md",
"rank": 2,
"score": 5.701635,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 4.079763,
"snippet_found": null
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 3.9514606,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 3.9501805,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 6,
"score": 3.8345904,
"snippet_found": null
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 3.5684028,
"snippet_found": null
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 3.4548547,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 3.4442132,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 3.2959082,
"snippet_found": null
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 3.278033,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 3.2729166,
"snippet_found": null
},
{
"filename": "transformations.md",
"rank": 13,
"score": 3.2519,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 14,
"score": 3.0763547,
"snippet_found": null
},
{
"filename": "invoices-and-payments.md",
"rank": 15,
"score": 2.9944477,
"snippet_found": null
},
{
"filename": "network-security.md",
"rank": 16,
"score": 2.795557,
"snippet_found": null
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 2.4639845,
"snippet_found": null
}
],
"expected_ranks": {
"delivery-retries.md": 5,
"troubleshooting.md": 1
},
"snippet_found": null,
"metrics": {
"recall@1": 0.5,
"recall@3": 0.5,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.6131471927654584,
"ndcg@5": 0.8503449055347546,
"ndcg@10": 0.8503449055347546
}
},
{
"id": "ordered-delivery",
"query": "Can I make sure events for one customer are delivered in sequence?",
"tags": [
"semantic"
],
"expected_files": [
"delivery-retries.md"
],
"expected_text": "ordering key",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 5.0055847,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 2,
"score": 4.8828163,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 3,
"score": 4.750148,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 4.722822,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 5,
"score": 4.638679,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 6,
"score": 4.568696,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 4.3880367,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 4.3245573,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 9,
"score": 4.2667556,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 10,
"score": 4.13976,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 11,
"score": 4.1162443,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 4.1093607,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 4.1089215,
"snippet_found": true
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 4.0251784,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 3.751133,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 16,
"score": 3.4920778,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 3.3043363,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 3.2784362,
"snippet_found": false
}
],
"expected_ranks": {
"delivery-retries.md": 3
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "destination-auto-disabled",
"query": "Why was my destination turned off without anyone touching it?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"delivery-retries.md",
"troubleshooting.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.720254,
"snippet_found": null
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 4.6855354,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 4.230507,
"snippet_found": null
},
{
"filename": "audit-log.md",
"rank": 4,
"score": 4.065977,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 3.9990711,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 6,
"score": 3.6033638,
"snippet_found": null
},
{
"filename": "network-security.md",
"rank": 7,
"score": 3.3606205,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 3.3185124,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 9,
"score": 3.0583892,
"snippet_found": null
},
{
"filename": "event-filtering.md",
"rank": 10,
"score": 2.7916946,
"snippet_found": null
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 2.7792022,
"snippet_found": null
},
{
"filename": "signing-secrets.md",
"rank": 12,
"score": 2.6849184,
"snippet_found": null
},
{
"filename": "roles-and-permissions.md",
"rank": 13,
"score": 2.385123,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 14,
"score": 2.2865496,
"snippet_found": null
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 2.131452,
"snippet_found": null
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.1214397,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 17,
"score": 2.0443506,
"snippet_found": null
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 1.9859804,
"snippet_found": null
}
],
"expected_ranks": {
"delivery-retries.md": 5,
"troubleshooting.md": 1
},
"snippet_found": null,
"metrics": {
"recall@1": 0.5,
"recall@3": 0.5,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.6131471927654584,
"ndcg@5": 0.8503449055347546,
"ndcg@10": 0.8503449055347546
}
},
{
"id": "dlq-what",
"query": "Where do events go once every delivery attempt has been used up?",
"tags": [
"semantic"
],
"expected_files": [
"dead-letter-queue.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 7.40966,
"snippet_found": null
},
{
"filename": "rate-limits.md",
"rank": 2,
"score": 6.4829597,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 3,
"score": 6.2448034,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 4,
"score": 5.8351073,
"snippet_found": null
},
{
"filename": "event-filtering.md",
"rank": 5,
"score": 5.679214,
"snippet_found": null
},
{
"filename": "transformations.md",
"rank": 6,
"score": 5.4656425,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 5.4533553,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 8,
"score": 5.332749,
"snippet_found": null
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 4.9943905,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 4.968727,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 4.9405565,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 4.714102,
"snippet_found": null
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 4.632801,
"snippet_found": null
},
{
"filename": "network-security.md",
"rank": 14,
"score": 4.599147,
"snippet_found": null
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 4.52356,
"snippet_found": null
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 4.458351,
"snippet_found": null
},
{
"filename": "roles-and-permissions.md",
"rank": 17,
"score": 4.3109264,
"snippet_found": null
},
{
"filename": "api-keys.md",
"rank": 18,
"score": 3.9744198,
"snippet_found": null
}
],
"expected_ranks": {
"dead-letter-queue.md": 1
},
"snippet_found": null,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0
}
},
{
"id": "dlq-retention-team",
"query": "On the Team plan, how long are undeliverable events kept before they are removed?",
"tags": [
"semantic"
],
"expected_files": [
"dead-letter-queue.md"
],
"expected_text": "14 days",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "data-retention.md",
"rank": 1,
"score": 7.8197613,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 2,
"score": 7.3345337,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 3,
"score": 7.332274,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 6.697984,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 5.410752,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 6,
"score": 5.0231695,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 4.924449,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 4.6471853,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 9,
"score": 4.3576174,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 10,
"score": 4.2551737,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 4.2262077,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 12,
"score": 4.0809183,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 3.9892564,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 14,
"score": 3.9197059,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 15,
"score": 3.8503838,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 16,
"score": 3.7696798,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 17,
"score": 3.5610816,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 3.4350631,
"snippet_found": false
}
],
"expected_ranks": {
"dead-letter-queue.md": 4
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.25,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.43067655807339306,
"ndcg@10": 0.43067655807339306,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "dlq-replay-other-destination",
"query": "How do I use rwctl dlq replay to resend failed events to a new endpoint?",
"tags": [
"exact-token"
],
"expected_files": [
"dead-letter-queue.md"
],
"expected_text": "dst_9x1aa",
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 8.870468,
"snippet_found": true
},
{
"filename": "cli-reference.md",
"rank": 2,
"score": 6.881461,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 5.751782,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 4,
"score": 5.043284,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 5,
"score": 4.7557015,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 4.6869016,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 7,
"score": 4.359108,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 8,
"score": 4.1611156,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 9,
"score": 4.0299006,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 3.9948087,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 11,
"score": 3.844854,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 3.5940793,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 13,
"score": 3.408236,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 3.3590999,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 3.3468688,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 3.2550454,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 17,
"score": 3.1481986,
"snippet_found": false
}
],
"expected_ranks": {
"dead-letter-queue.md": 1
},
"snippet_found": true,
"passage_rank": 2,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.5
}
},
{
"id": "replay-header",
"query": "How can my endpoint tell that a delivery is a replay?",
"tags": [
"semantic"
],
"expected_files": [
"dead-letter-queue.md"
],
"expected_text": "Relaywise-Replay",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 5.8395176,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 5.7776,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 4.659917,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 4,
"score": 4.5585823,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 4.535796,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 6,
"score": 4.4854813,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 7,
"score": 4.418273,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 4.399255,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 4.092454,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 10,
"score": 3.7910876,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 11,
"score": 3.6927078,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 12,
"score": 3.5761201,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 3.4670155,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 14,
"score": 3.373952,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 3.302986,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 3.2588396,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 3.145359,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 18,
"score": 3.0399675,
"snippet_found": false
}
],
"expected_ranks": {
"dead-letter-queue.md": 1
},
"snippet_found": true,
"passage_rank": 10,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.1
}
},
{
"id": "ingest-rate",
"query": "How fast am I allowed to send events on each plan?",
"tags": [
"semantic"
],
"expected_files": [
"rate-limits.md"
],
"expected_text": "events per second",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "rate-limits.md",
"rank": 1,
"score": 6.559215,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 2,
"score": 6.377366,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 6.3756666,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 4,
"score": 5.5507555,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 5,
"score": 4.7628126,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 6,
"score": 4.5803623,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 7,
"score": 4.3698053,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 8,
"score": 4.1771226,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 9,
"score": 4.1309533,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 10,
"score": 4.102806,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 11,
"score": 3.8547676,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 12,
"score": 3.605217,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 13,
"score": 3.5623853,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 3.453558,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 15,
"score": 3.3093355,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 16,
"score": 3.2472394,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 17,
"score": 3.2431312,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 3.1259098,
"snippet_found": false
}
],
"expected_ranks": {
"rate-limits.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "err-4290",
"query": "RW-4290",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"rate-limits.md",
"error-codes.md"
],
"expected_text": "RW-4290",
"chunks_returned": 8,
"unique_files_returned": 2,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 5.104968,
"snippet_found": true
},
{
"filename": "rate-limits.md",
"rank": 2,
"score": 4.61705,
"snippet_found": true
}
],
"expected_ranks": {
"rate-limits.md": 2,
"error-codes.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "payload-max-size",
"query": "What is the largest event body I can send?",
"tags": [
"semantic"
],
"expected_files": [
"rate-limits.md"
],
"expected_text": "256 KB",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.1728625,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 2,
"score": 4.963256,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 4.2940288,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 4,
"score": 3.8478184,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 5,
"score": 3.7793765,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 6,
"score": 3.7538767,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 3.7113836,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 8,
"score": 3.647768,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 9,
"score": 3.63453,
"snippet_found": true
},
{
"filename": "signing-secrets.md",
"rank": 10,
"score": 3.572057,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 3.561708,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 12,
"score": 3.4531813,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 3.378756,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 14,
"score": 3.3345623,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 3.2072341,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 16,
"score": 3.0846722,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.9861572,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 18,
"score": 2.791448,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 2.7770343,
"snippet_found": false
}
],
"expected_ranks": {
"rate-limits.md": 9
},
"snippet_found": true,
"passage_rank": 14,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.0,
"recall@10": 1.0,
"reciprocal_rank": 0.1111111111111111,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.0,
"ndcg@10": 0.3010299956639812,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.07142857142857142
}
},
{
"id": "batch-size",
"query": "How many events fit in one batch request?",
"tags": [
"keyword"
],
"expected_files": [
"rate-limits.md"
],
"expected_text": "500 events",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "rate-limits.md",
"rank": 1,
"score": 6.440647,
"snippet_found": true
},
{
"filename": "event-filtering.md",
"rank": 2,
"score": 4.471184,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 3,
"score": 4.471135,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 4.4559145,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 5,
"score": 4.4337754,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 6,
"score": 4.210376,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 7,
"score": 4.1672397,
"snippet_found": true
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.9377017,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 9,
"score": 3.857754,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 10,
"score": 3.7983751,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 3.741257,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 12,
"score": 3.216155,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 13,
"score": 3.2107887,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 14,
"score": 3.144632,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 15,
"score": 2.9834402,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 16,
"score": 2.9600403,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 2.867922,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 18,
"score": 2.8529837,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 19,
"score": 2.5421212,
"snippet_found": false
}
],
"expected_ranks": {
"rate-limits.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "err-3002",
"query": "What does error RW-3002 mean?",
"tags": [
"exact-token"
],
"expected_files": [
"error-codes.md"
],
"expected_text": "TLS handshake failed",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 7.1374054,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 4.6004024,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 3,
"score": 3.5720081,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 4,
"score": 3.5708163,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 3.373531,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 3.3219764,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 7,
"score": 3.270485,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 8,
"score": 3.0856066,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 2.9129028,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 2.8195572,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 11,
"score": 2.7496767,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 12,
"score": 2.6049972,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 2.339275,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 14,
"score": 2.285453,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 15,
"score": 2.267887,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 16,
"score": 2.1199887,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 1.9273345,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 1.7775464,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 1.6508594,
"snippet_found": false
}
],
"expected_ranks": {
"error-codes.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "err-5004",
"query": "RW-5004",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"error-codes.md",
"transformations.md"
],
"expected_text": "RW-5004",
"chunks_returned": 8,
"unique_files_returned": 2,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 7.083034,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 2,
"score": 4.1704693,
"snippet_found": true
}
],
"expected_ranks": {
"error-codes.md": 1,
"transformations.md": 2
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "err-2031",
"query": "Why was my destination URL rejected with RW-2031?",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"error-codes.md",
"network-security.md"
],
"expected_text": "RW-2031",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 7.7312093,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 5.75021,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 4.6584964,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.269464,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 4.031956,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 3.6961586,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 3.5220945,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 8,
"score": 3.4944685,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 9,
"score": 3.3147504,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 3.3085356,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 11,
"score": 3.0455709,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 3.013853,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 13,
"score": 3.0125816,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 3.006339,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 15,
"score": 2.862323,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.804049,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 2.4442608,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 18,
"score": 2.4357595,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 2.4208562,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 20,
"score": 2.021038,
"snippet_found": false
}
],
"expected_ranks": {
"error-codes.md": 1,
"network-security.md": 2
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "err-7001",
"query": "What causes RW-7001?",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"error-codes.md",
"regions.md"
],
"expected_text": "RW-7001",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 4.55962,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 2,
"score": 3.2900271,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 3,
"score": 3.231194,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 3.0057564,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 2.8333864,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 6,
"score": 2.6037674,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 2.4505215,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 8,
"score": 2.2581377,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 9,
"score": 2.2200274,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 2.1641731,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 2.1146564,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 2.0174422,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 1.6311694,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 14,
"score": 1.4668834,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 15,
"score": 1.456635,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 1.4466288,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 17,
"score": 1.4390881,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 1.4314262,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 1.4135953,
"snippet_found": false
}
],
"expected_ranks": {
"error-codes.md": 1,
"regions.md": 3
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.9197207891481876,
"ndcg@5": 0.9197207891481876,
"ndcg@10": 0.9197207891481876,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "cfg-concurrency",
"query": "Which setting limits how many requests hit my endpoint in parallel?",
"tags": [
"semantic"
],
"expected_files": [
"configuration-reference.md"
],
"expected_text": "delivery.max_concurrency",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "configuration-reference.md",
"rank": 1,
"score": 6.5323057,
"snippet_found": true
},
{
"filename": "rate-limits.md",
"rank": 2,
"score": 5.4767127,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 4.882698,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 4,
"score": 4.4162087,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 4.319521,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 6,
"score": 3.912937,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 7,
"score": 3.9068096,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 8,
"score": 3.5763764,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 9,
"score": 3.4460037,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 10,
"score": 3.442761,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 11,
"score": 3.3418725,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 3.2560182,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 13,
"score": 3.2276452,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 14,
"score": 3.1950235,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 15,
"score": 3.0526667,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 16,
"score": 2.9480395,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 17,
"score": 2.910755,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 2.5600786,
"snippet_found": false
}
],
"expected_ranks": {
"configuration-reference.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "cfg-dedupe-window",
"query": "ingest.dedupe_window_seconds",
"tags": [
"exact-token"
],
"expected_files": [
"configuration-reference.md"
],
"expected_text": "ingest.dedupe_window_seconds",
"chunks_returned": 8,
"unique_files_returned": 1,
"ranked_files": [
{
"filename": "configuration-reference.md",
"rank": 1,
"score": 3.756914,
"snippet_found": true
}
],
"expected_ranks": {
"configuration-reference.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "cfg-explain",
"query": "How do I find out which level a setting's value is coming from?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"configuration-reference.md",
"cli-reference.md"
],
"expected_text": "rwctl config explain",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "configuration-reference.md",
"rank": 1,
"score": 5.1910233,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 5.1194305,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 3,
"score": 4.7757354,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 4,
"score": 4.3601513,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 4.301612,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 6,
"score": 4.265447,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 3.9866903,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 8,
"score": 3.753189,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 3.693151,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 10,
"score": 3.283983,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 3.2680726,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 12,
"score": 3.261208,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 3.0610666,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 2.967939,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 15,
"score": 2.8458738,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 16,
"score": 2.8134208,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.7762527,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 18,
"score": 2.6981857,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 2.6218233,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 20,
"score": 2.5113726,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 21,
"score": 2.4949827,
"snippet_found": false
}
],
"expected_ranks": {
"configuration-reference.md": 1,
"cli-reference.md": 8
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.5,
"recall@3": 0.5,
"recall@5": 0.5,
"recall@10": 0.5,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.6131471927654584,
"ndcg@5": 0.6131471927654584,
"ndcg@10": 0.6131471927654584,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "schema-validation",
"query": "Can incoming payloads be checked against a schema and refused when they don't match?",
"tags": [
"semantic"
],
"expected_files": [
"configuration-reference.md"
],
"expected_text": "ingest.schema_validation",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "configuration-reference.md",
"rank": 1,
"score": 7.2857604,
"snippet_found": true
},
{
"filename": "invoices-and-payments.md",
"rank": 2,
"score": 4.1829777,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 3.9681184,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 3.9628394,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 3.8958888,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 6,
"score": 3.8805716,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 3.7766762,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 8,
"score": 3.7272916,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 3.6178882,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 10,
"score": 3.5861323,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 11,
"score": 3.5555618,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 12,
"score": 3.5408092,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 13,
"score": 3.2773092,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 3.2318847,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 15,
"score": 3.1293461,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 16,
"score": 3.0882106,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 17,
"score": 2.9718976,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 18,
"score": 2.8200388,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 19,
"score": 2.7177334,
"snippet_found": false
}
],
"expected_ranks": {
"configuration-reference.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "filter-wildcard",
"query": "How do I send every kind of invoice event, including nested types, to one endpoint?",
"tags": [
"semantic"
],
"expected_files": [
"event-filtering.md"
],
"expected_text": "invoice.**",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 6.599655,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 2,
"score": 5.760765,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 3,
"score": 5.569148,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 4,
"score": 5.527859,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 5,
"score": 5.472649,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 6,
"score": 5.421536,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 5.261847,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 8,
"score": 5.023038,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 9,
"score": 4.937884,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 4.873948,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 4.553656,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 12,
"score": 4.50159,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 13,
"score": 4.4150305,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 4.278399,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 3.8844829,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 16,
"score": 3.8552125,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 17,
"score": 3.8130789,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 3.5139594,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 19,
"score": 3.3147485,
"snippet_found": false
}
],
"expected_ranks": {
"event-filtering.md": 7
},
"snippet_found": true,
"passage_rank": 43,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.0,
"recall@10": 1.0,
"reciprocal_rank": 0.14285714285714285,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.0,
"ndcg@10": 0.3333333333333333,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.023255813953488372
}
},
{
"id": "filter-missing-field",
"query": "What happens when a filter refers to a field the event doesn't have?",
"tags": [
"semantic"
],
"expected_files": [
"event-filtering.md"
],
"expected_text": "it is not an error",
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "event-filtering.md",
"rank": 1,
"score": 6.3981233,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 2,
"score": 6.132348,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 3,
"score": 5.273807,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 4.9314632,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 4.871355,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 6,
"score": 4.5469966,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 4.467554,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 4.247962,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 3.781559,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 10,
"score": 3.587966,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 11,
"score": 3.5403461,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 12,
"score": 3.4803455,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 13,
"score": 3.4259849,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 14,
"score": 3.240863,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 3.217833,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 16,
"score": 3.142176,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 17,
"score": 3.012438,
"snippet_found": false
}
],
"expected_ranks": {
"event-filtering.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "unrouted-events",
"query": "What happens to an event that no route picks up?",
"tags": [
"semantic"
],
"expected_files": [
"event-filtering.md"
],
"expected_text": "unrouted",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "event-filtering.md",
"rank": 1,
"score": 6.7026176,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 6.576373,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 3,
"score": 6.105928,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 5.184962,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 4.8381486,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 6,
"score": 4.526141,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 4.250954,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 8,
"score": 4.041794,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 4.0392456,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 3.9502084,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 3.8146112,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 3.8039114,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 13,
"score": 3.4500139,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 14,
"score": 3.4208,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 15,
"score": 3.2320712,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 16,
"score": 3.1246333,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 3.1182313,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 3.030007,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 19,
"score": 2.9305377,
"snippet_found": false
}
],
"expected_ranks": {
"event-filtering.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "route-fan-out",
"query": "How many endpoints can one route deliver to?",
"tags": [
"semantic"
],
"expected_files": [
"event-filtering.md"
],
"expected_text": "up to 20 destinations",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 6.023015,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 2,
"score": 5.745181,
"snippet_found": true
},
{
"filename": "configuration-reference.md",
"rank": 3,
"score": 5.1195517,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 4.844228,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 5,
"score": 4.83197,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 6,
"score": 4.238139,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 7,
"score": 4.2260113,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 4.079329,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 3.8717012,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 10,
"score": 3.8605428,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 11,
"score": 3.7901826,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 12,
"score": 3.5144677,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 13,
"score": 3.5060587,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 14,
"score": 3.367303,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 15,
"score": 3.3218155,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 3.022322,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 17,
"score": 2.9369514,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 18,
"score": 2.922502,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 2.7017128,
"snippet_found": false
}
],
"expected_ranks": {
"event-filtering.md": 2
},
"snippet_found": true,
"passage_rank": 2,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.5,
"ndcg@1": 0.0,
"ndcg@3": 0.6309297535714575,
"ndcg@5": 0.6309297535714575,
"ndcg@10": 0.6309297535714575,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.5
}
},
{
"id": "strip-fields-before-delivery",
"query": "How can I strip card numbers out of events before they are forwarded to a partner?",
"tags": [
"semantic"
],
"expected_files": [
"transformations.md"
],
"expected_text": "card_number",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 5.678779,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 2,
"score": 4.841915,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 4.5332537,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 4.3789315,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 5,
"score": 4.374852,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 6,
"score": 4.303219,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 4.297907,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 4.2757826,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 9,
"score": 4.0928054,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 10,
"score": 4.013118,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 3.958882,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 12,
"score": 3.8992069,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 13,
"score": 3.830959,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 3.8095558,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 15,
"score": 3.6439047,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 16,
"score": 3.4828746,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 17,
"score": 3.4731426,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 18,
"score": 3.339641,
"snippet_found": true
},
{
"filename": "roles-and-permissions.md",
"rank": 19,
"score": 3.257053,
"snippet_found": false
}
],
"expected_ranks": {
"transformations.md": 18
},
"snippet_found": true,
"passage_rank": 44,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.0,
"recall@10": 0.0,
"reciprocal_rank": 0.05555555555555555,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.0,
"ndcg@10": 0.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.022727272727272728
}
},
{
"id": "transform-network",
"query": "Can transformation code call an external API?",
"tags": [
"semantic"
],
"expected_files": [
"transformations.md"
],
"expected_text": "No network access",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "error-codes.md",
"rank": 1,
"score": 4.3027563,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 2,
"score": 3.7211428,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 3,
"score": 3.5431662,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 4,
"score": 3.4637728,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 5,
"score": 3.4438903,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 6,
"score": 3.3542087,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 7,
"score": 3.1996362,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 3.1790874,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 9,
"score": 3.1691957,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 10,
"score": 2.7622945,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 11,
"score": 2.7219043,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 12,
"score": 2.7049937,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 2.6479259,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 14,
"score": 2.5942392,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 15,
"score": 2.5000453,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 2.412778,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 2.2223032,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 2.120767,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 2.0920844,
"snippet_found": false
}
],
"expected_ranks": {
"transformations.md": 3
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "transform-rollback",
"query": "I shipped a broken transformation, how do I go back to the previous one?",
"tags": [
"semantic"
],
"expected_files": [
"transformations.md"
],
"expected_text": "Make active",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.23054,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 2,
"score": 4.2707195,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 3,
"score": 4.0321445,
"snippet_found": true
},
{
"filename": "rate-limits.md",
"rank": 4,
"score": 3.5525649,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 3.4241276,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 6,
"score": 3.4105673,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 3.4089398,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.4029415,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 3.3298483,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 3.2877781,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 11,
"score": 3.2613475,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 3.2458348,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 3.2236872,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 14,
"score": 3.2179403,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 15,
"score": 3.1666298,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 16,
"score": 2.8349867,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 2.7176538,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 18,
"score": 2.5744977,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 2.480778,
"snippet_found": false
}
],
"expected_ranks": {
"transformations.md": 3
},
"snippet_found": true,
"passage_rank": 23,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.043478260869565216
}
},
{
"id": "plan-scale-price",
"query": "What does the Scale plan cost and how many events come with it?",
"tags": [
"keyword"
],
"expected_files": [
"billing-plans.md"
],
"expected_text": "20 million",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "data-retention.md",
"rank": 1,
"score": 6.165932,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 2,
"score": 5.6396055,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 3,
"score": 5.595186,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.96609,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 4.8011065,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 4.7937565,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 4.7090893,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 4.6951523,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 9,
"score": 4.6432734,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 10,
"score": 4.542534,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 11,
"score": 3.9930236,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 3.9314911,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 3.7961717,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 14,
"score": 3.7279844,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 3.7183979,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 16,
"score": 3.5648823,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 17,
"score": 3.5081198,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 18,
"score": 3.369672,
"snippet_found": false
}
],
"expected_ranks": {
"billing-plans.md": 3
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "over-quota",
"query": "What happens if we send more events in a month than our plan includes?",
"tags": [
"semantic"
],
"expected_files": [
"billing-plans.md"
],
"expected_text": "billed as overage",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "billing-plans.md",
"rank": 1,
"score": 6.611384,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 5.573196,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 5.069791,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 4,
"score": 4.896354,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 5,
"score": 4.8308477,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 6,
"score": 4.6927853,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 7,
"score": 4.615057,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 8,
"score": 4.568859,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 9,
"score": 4.510277,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 4.4583926,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 4.3350053,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 4.3333607,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 13,
"score": 4.2085896,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 3.864415,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 15,
"score": 3.6676457,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 16,
"score": 3.6663413,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 17,
"score": 3.626868,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 18,
"score": 3.5489278,
"snippet_found": false
}
],
"expected_ranks": {
"billing-plans.md": 1
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "spending-cap",
"query": "How do I make sure we never get a surprise bill?",
"tags": [
"semantic"
],
"expected_files": [
"billing-plans.md"
],
"expected_text": "hard cap",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "cli-reference.md",
"rank": 1,
"score": 4.176035,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 2,
"score": 3.7238424,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 3,
"score": 3.373507,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 4,
"score": 3.3062384,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 3.1803827,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 2.8217483,
"snippet_found": true
},
{
"filename": "signing-secrets.md",
"rank": 7,
"score": 2.8194714,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 2.7950668,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 9,
"score": 2.7918391,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 10,
"score": 2.7697334,
"snippet_found": true
},
{
"filename": "roles-and-permissions.md",
"rank": 11,
"score": 2.3493757,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 12,
"score": 2.2208354,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 13,
"score": 2.1967921,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 14,
"score": 2.0774598,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 15,
"score": 2.0550961,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 16,
"score": 2.0002537,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 17,
"score": 1.9782275,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 18,
"score": 1.8935627,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 19,
"score": 1.8432963,
"snippet_found": false
}
],
"expected_ranks": {
"billing-plans.md": 10
},
"snippet_found": true,
"passage_rank": 11,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.0,
"recall@10": 1.0,
"reciprocal_rank": 0.1,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.0,
"ndcg@10": 0.2890648263178879,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.09090909090909091
}
},
{
"id": "downgrade-timing",
"query": "When does a plan downgrade kick in?",
"tags": [
"semantic"
],
"expected_files": [
"billing-plans.md"
],
"expected_text": "end of the current billing period",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "billing-plans.md",
"rank": 1,
"score": 4.5011973,
"snippet_found": true
},
{
"filename": "invoices-and-payments.md",
"rank": 2,
"score": 3.9330368,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 3.6819746,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 4,
"score": 3.3807042,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 5,
"score": 3.145202,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 6,
"score": 2.9044037,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 2.6842203,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 8,
"score": 2.652769,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 9,
"score": 2.559432,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 10,
"score": 2.5516438,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 2.5177975,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 12,
"score": 2.4889445,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 2.4745228,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 14,
"score": 2.397379,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 2.3737328,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.3688858,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 17,
"score": 2.2727196,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 18,
"score": 2.255967,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 19,
"score": 2.1137109,
"snippet_found": false
}
],
"expected_ranks": {
"billing-plans.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "nonprofit-discount",
"query": "Is there a discount for nonprofits?",
"tags": [
"semantic"
],
"expected_files": [
"billing-plans.md"
],
"expected_text": "50 percent off",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 3.192665,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 2,
"score": 3.1329155,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 2.7976594,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 4,
"score": 2.500914,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 2.431467,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 6,
"score": 2.3387825,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 7,
"score": 2.1462817,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 2.138791,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 2.0935802,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 1.9763438,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 11,
"score": 1.9572263,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 12,
"score": 1.9540287,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 1.8847063,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 14,
"score": 1.8813934,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 1.8805516,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 16,
"score": 1.8092102,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 17,
"score": 1.799421,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 1.7747855,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 1.7663033,
"snippet_found": false
}
],
"expected_ranks": {
"billing-plans.md": 2
},
"snippet_found": true,
"passage_rank": 2,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.5,
"ndcg@1": 0.0,
"ndcg@3": 0.6309297535714575,
"ndcg@5": 0.6309297535714575,
"ndcg@10": 0.6309297535714575,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.5
}
},
{
"id": "card-declined",
"query": "What happens if our card is declined?",
"tags": [
"semantic"
],
"expected_files": [
"invoices-and-payments.md"
],
"expected_text": "suspended",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 4.5424986,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 2,
"score": 3.1390238,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 3,
"score": 2.9242349,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 2.853714,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 2.7698298,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 6,
"score": 2.6912606,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 2.6068764,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 8,
"score": 2.4017735,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 9,
"score": 2.3534894,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 10,
"score": 2.3194094,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 2.2925973,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 2.2087455,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 2.170509,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 14,
"score": 2.1145196,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 2.0395908,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 16,
"score": 1.8723415,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 17,
"score": 1.8695025,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 18,
"score": 1.8511734,
"snippet_found": false
}
],
"expected_ranks": {
"invoices-and-payments.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "invoice-download",
"query": "Where can I download past invoices?",
"tags": [
"keyword"
],
"expected_files": [
"invoices-and-payments.md"
],
"expected_text": "PDF",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 5.479479,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 3.3870986,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 3,
"score": 3.3651319,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 4,
"score": 3.1572304,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 2.869647,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 6,
"score": 2.8024464,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 7,
"score": 2.6480045,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 2.558901,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 9,
"score": 2.514241,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 10,
"score": 2.4659653,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 2.4333763,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 12,
"score": 2.4234314,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 13,
"score": 2.3397257,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 14,
"score": 2.3184114,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.2858236,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 2.261516,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 17,
"score": 2.2397711,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 18,
"score": 1.9229008,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 19,
"score": 1.9125963,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 20,
"score": 1.8740361,
"snippet_found": false
}
],
"expected_ranks": {
"invoices-and-payments.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "annual-refund",
"query": "Can I get my money back on an annual subscription?",
"tags": [
"semantic"
],
"expected_files": [
"invoices-and-payments.md"
],
"expected_text": "30 days of purchase",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 4.030309,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 2,
"score": 3.4916003,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 3,
"score": 3.1594803,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 3.083531,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 2.862523,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 2.7023883,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 7,
"score": 2.6665974,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 8,
"score": 2.5352874,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 9,
"score": 2.4236565,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.3765519,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 11,
"score": 2.3002856,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 12,
"score": 2.2416017,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 2.06355,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 14,
"score": 1.9668009,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 15,
"score": 1.899621,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 1.8948866,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 17,
"score": 1.8911364,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 1.875198,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 19,
"score": 1.7308294,
"snippet_found": false
}
],
"expected_ranks": {
"invoices-and-payments.md": 3
},
"snippet_found": true,
"passage_rank": 39,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.02564102564102564
}
},
{
"id": "pay-by-wire",
"query": "Can we pay by wire instead of by card?",
"tags": [
"semantic"
],
"expected_files": [
"invoices-and-payments.md"
],
"expected_text": "bank transfer",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 5.018153,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 3.2159462,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 3.201638,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 4,
"score": 3.147363,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 5,
"score": 2.6815596,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 6,
"score": 2.6559532,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 7,
"score": 2.5634036,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 8,
"score": 2.364353,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 9,
"score": 2.3472905,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 10,
"score": 2.2389264,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 2.2166698,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 12,
"score": 2.180029,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 2.1364298,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 14,
"score": 2.1318357,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 15,
"score": 2.0653362,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 16,
"score": 1.992836,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 17,
"score": 1.948524,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 18,
"score": 1.9307742,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 19,
"score": 1.7839562,
"snippet_found": false
}
],
"expected_ranks": {
"invoices-and-payments.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "payload-retention-scale",
"query": "How long are event bodies stored on the Scale plan?",
"tags": [
"semantic"
],
"expected_files": [
"data-retention.md"
],
"expected_text": "These are maximums",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "data-retention.md",
"rank": 1,
"score": 7.134757,
"snippet_found": true
},
{
"filename": "billing-plans.md",
"rank": 2,
"score": 5.646868,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 3,
"score": 5.632083,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 4,
"score": 5.5159082,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 5,
"score": 5.322894,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 6,
"score": 4.917618,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 4.646406,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 8,
"score": 4.593455,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 9,
"score": 4.1423883,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 10,
"score": 4.102504,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 11,
"score": 3.5780063,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 12,
"score": 3.4474115,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 3.1836896,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 3.1513236,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 15,
"score": 3.037868,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 16,
"score": 2.9461775,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.7285311,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 18,
"score": 2.6023915,
"snippet_found": false
}
],
"expected_ranks": {
"data-retention.md": 1
},
"snippet_found": true,
"passage_rank": 14,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.07142857142857142
}
},
{
"id": "erasure-request",
"query": "A customer asked us to erase their personal data, how do I remove their events?",
"tags": [
"semantic"
],
"expected_files": [
"data-retention.md"
],
"expected_text": "Delete events",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "data-retention.md",
"rank": 1,
"score": 7.6901236,
"snippet_found": true
},
{
"filename": "roles-and-permissions.md",
"rank": 2,
"score": 6.390702,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 3,
"score": 5.357597,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 4,
"score": 4.2579355,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 5,
"score": 4.2515087,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 6,
"score": 4.1859593,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 4.102975,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 8,
"score": 3.948564,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 9,
"score": 3.9145777,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 10,
"score": 3.8292284,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 3.7746987,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 12,
"score": 3.7034235,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 13,
"score": 3.6457257,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 3.4510388,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 3.381026,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 16,
"score": 3.1673608,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 17,
"score": 3.1267197,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 18,
"score": 2.711932,
"snippet_found": false
}
],
"expected_ranks": {
"data-retention.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "never-store-field",
"query": "How do I keep a sensitive field from ever being saved?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"data-retention.md",
"configuration-reference.md"
],
"expected_text": "retention.redact_fields",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "transformations.md",
"rank": 1,
"score": 4.269417,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 2,
"score": 4.132926,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 4.115541,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 4,
"score": 3.9203172,
"snippet_found": true
},
{
"filename": "single-sign-on.md",
"rank": 5,
"score": 3.7807415,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 6,
"score": 3.7652113,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 3.6132348,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.4858923,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 9,
"score": 3.1855412,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 10,
"score": 3.0955997,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 11,
"score": 3.075557,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 12,
"score": 2.8341694,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 13,
"score": 2.6905017,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 14,
"score": 2.600214,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.5657752,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 2.553431,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 17,
"score": 2.3961258,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 18,
"score": 2.3078952,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 2.2757673,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 20,
"score": 2.2442474,
"snippet_found": false
}
],
"expected_ranks": {
"data-retention.md": 4,
"configuration-reference.md": 17
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.5,
"recall@10": 0.5,
"reciprocal_rank": 0.25,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.2640681225725909,
"ndcg@10": 0.2640681225725909,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "export-to-own-storage",
"query": "Can I keep events for longer than the plan allows by copying them to storage we own?",
"tags": [
"semantic"
],
"expected_files": [
"data-retention.md"
],
"expected_text": "scheduled export",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "data-retention.md",
"rank": 1,
"score": 7.004419,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 5.72364,
"snippet_found": true
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 5.051481,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 4,
"score": 4.8311186,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 4.7930336,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 6,
"score": 4.558029,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 7,
"score": 4.422286,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 8,
"score": 4.2385845,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 4.049222,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 10,
"score": 4.0140843,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 3.8202207,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 12,
"score": 3.7686496,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 13,
"score": 3.7210777,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 14,
"score": 3.7047937,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 3.7034209,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 16,
"score": 3.6294847,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 17,
"score": 3.6183372,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 3.476426,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 19,
"score": 3.2785215,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 20,
"score": 3.2107797,
"snippet_found": false
}
],
"expected_ranks": {
"data-retention.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "replay-event-gone",
"query": "Replay fails saying the event isn't available any more, why?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"data-retention.md",
"troubleshooting.md"
],
"expected_text": null,
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.111964,
"snippet_found": null
},
{
"filename": "delivery-retries.md",
"rank": 2,
"score": 4.6264553,
"snippet_found": null
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 4.382534,
"snippet_found": null
},
{
"filename": "error-codes.md",
"rank": 4,
"score": 4.204933,
"snippet_found": null
},
{
"filename": "event-filtering.md",
"rank": 5,
"score": 4.008878,
"snippet_found": null
},
{
"filename": "transformations.md",
"rank": 6,
"score": 4.000078,
"snippet_found": null
},
{
"filename": "signing-secrets.md",
"rank": 7,
"score": 3.8352492,
"snippet_found": null
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 3.6457763,
"snippet_found": null
},
{
"filename": "data-retention.md",
"rank": 9,
"score": 3.5607314,
"snippet_found": null
},
{
"filename": "billing-plans.md",
"rank": 10,
"score": 3.5086422,
"snippet_found": null
},
{
"filename": "invoices-and-payments.md",
"rank": 11,
"score": 3.5036902,
"snippet_found": null
},
{
"filename": "configuration-reference.md",
"rank": 12,
"score": 3.4338822,
"snippet_found": null
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 3.0842357,
"snippet_found": null
},
{
"filename": "regions.md",
"rank": 14,
"score": 3.065458,
"snippet_found": null
},
{
"filename": "network-security.md",
"rank": 15,
"score": 2.9612713,
"snippet_found": null
},
{
"filename": "getting-started.md",
"rank": 16,
"score": 2.9603195,
"snippet_found": null
},
{
"filename": "roles-and-permissions.md",
"rank": 17,
"score": 2.8865094,
"snippet_found": null
},
{
"filename": "api-keys.md",
"rank": 18,
"score": 2.8189383,
"snippet_found": null
}
],
"expected_ranks": {
"data-retention.md": 9,
"troubleshooting.md": 1
},
"snippet_found": null,
"metrics": {
"recall@1": 0.5,
"recall@3": 0.5,
"recall@5": 0.5,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 0.6131471927654584,
"ndcg@5": 0.6131471927654584,
"ndcg@10": 0.7977228895450266
}
},
{
"id": "eu-residency",
"query": "We need our data to stay inside the EU, is that possible?",
"tags": [
"semantic"
],
"expected_files": [
"regions.md"
],
"expected_text": "eu-central",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "regions.md",
"rank": 1,
"score": 5.606211,
"snippet_found": true
},
{
"filename": "api-keys.md",
"rank": 2,
"score": 3.7901576,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 3,
"score": 3.3422961,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 4,
"score": 3.0132427,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 5,
"score": 2.9385717,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 6,
"score": 2.8419957,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 7,
"score": 2.7693527,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 8,
"score": 2.7328167,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 9,
"score": 2.4419742,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 2.4058669,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 11,
"score": 2.3926773,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 12,
"score": 2.3721557,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 13,
"score": 2.3270187,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 2.2621386,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 2.2538724,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 16,
"score": 2.1800587,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 17,
"score": 2.1209936,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 18,
"score": 2.111018,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 19,
"score": 1.9036365,
"snippet_found": false
}
],
"expected_ranks": {
"regions.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "move-region",
"query": "Can I move an existing workspace to another region?",
"tags": [
"semantic"
],
"expected_files": [
"regions.md"
],
"expected_text": "cannot be changed",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "regions.md",
"rank": 1,
"score": 7.4162574,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.9218082,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 4.279954,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 4,
"score": 3.787121,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 5,
"score": 3.5782912,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 6,
"score": 3.4162853,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 3.3293962,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 8,
"score": 3.0762732,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 3.0027943,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.9621854,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 11,
"score": 2.8567605,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 2.7653418,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 13,
"score": 2.742486,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 14,
"score": 2.7398138,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.7382836,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 16,
"score": 2.6832218,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 17,
"score": 2.652599,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 18,
"score": 2.6192703,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 19,
"score": 2.2097597,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 20,
"score": 2.1313481,
"snippet_found": false
}
],
"expected_ranks": {
"regions.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "firewall-ips",
"query": "Which IP addresses do I need to let through our firewall so deliveries arrive?",
"tags": [
"semantic",
"multi-doc"
],
"expected_files": [
"regions.md",
"network-security.md"
],
"expected_text": "rwctl network egress-ips",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "regions.md",
"rank": 1,
"score": 7.6239576,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 7.329286,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 4.260872,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 4,
"score": 3.736021,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 5,
"score": 3.5849426,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 6,
"score": 3.4546218,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 7,
"score": 3.4361572,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 8,
"score": 3.4263103,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 9,
"score": 3.3875096,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 10,
"score": 3.3755388,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 11,
"score": 3.2378645,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 12,
"score": 3.1179535,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 13,
"score": 3.0980186,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 14,
"score": 3.062451,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 15,
"score": 3.0315938,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 16,
"score": 2.801557,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 2.7423558,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 18,
"score": 2.728594,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 19,
"score": 2.5501351,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 20,
"score": 2.5037901,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 21,
"score": 2.3320217,
"snippet_found": false
}
],
"expected_ranks": {
"regions.md": 1,
"network-security.md": 2
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "alert-on-failing-endpoint",
"query": "How do I get notified when an endpoint starts failing?",
"tags": [
"semantic"
],
"expected_files": [
"monitoring-and-alerts.md"
],
"expected_text": "delivery.success_rate",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "monitoring-and-alerts.md",
"rank": 1,
"score": 4.7551217,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 2,
"score": 4.4837484,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 3.97679,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 4,
"score": 3.7504718,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 3.7289722,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 6,
"score": 3.680981,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 3.4579203,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 3.360943,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 9,
"score": 3.2502306,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 10,
"score": 3.0607262,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 11,
"score": 3.031942,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 12,
"score": 2.939257,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 13,
"score": 2.9016738,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 14,
"score": 2.7974367,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 15,
"score": 2.660655,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.4786308,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.3564715,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 2.2467048,
"snippet_found": false
}
],
"expected_ranks": {
"monitoring-and-alerts.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "metric-dlq-depth",
"query": "dlq.depth",
"tags": [
"exact-token",
"multi-doc"
],
"expected_files": [
"monitoring-and-alerts.md",
"dead-letter-queue.md"
],
"expected_text": "dlq.depth",
"chunks_returned": 8,
"unique_files_returned": 2,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 3.1213505,
"snippet_found": true
},
{
"filename": "monitoring-and-alerts.md",
"rank": 2,
"score": 2.994694,
"snippet_found": true
}
],
"expected_ranks": {
"monitoring-and-alerts.md": 2,
"dead-letter-queue.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "alert-noise",
"query": "How do I stop an alert from notifying us over and over?",
"tags": [
"semantic"
],
"expected_files": [
"monitoring-and-alerts.md"
],
"expected_text": "cooldown",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "monitoring-and-alerts.md",
"rank": 1,
"score": 5.3605433,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 2,
"score": 4.4922824,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 3,
"score": 3.8867335,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 4,
"score": 3.680799,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 3.500159,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 6,
"score": 3.3970802,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 7,
"score": 3.2065027,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 8,
"score": 3.1697018,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 3.005413,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 10,
"score": 2.8459678,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 11,
"score": 2.7075675,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 12,
"score": 2.6995163,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 2.6899762,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 2.6840582,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 15,
"score": 2.664556,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.4973686,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.4592614,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 18,
"score": 2.4189043,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 19,
"score": 2.3941422,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 20,
"score": 2.3926606,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 21,
"score": 2.2645342,
"snippet_found": false
}
],
"expected_ranks": {
"monitoring-and-alerts.md": 1
},
"snippet_found": true,
"passage_rank": 17,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.058823529411764705
}
},
{
"id": "metrics-history",
"query": "How far back do the charts go?",
"tags": [
"semantic"
],
"expected_files": [
"monitoring-and-alerts.md"
],
"expected_text": "13 months",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "monitoring-and-alerts.md",
"rank": 1,
"score": 3.7359376,
"snippet_found": true
},
{
"filename": "invoices-and-payments.md",
"rank": 2,
"score": 3.1819804,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 3,
"score": 2.9821012,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 4,
"score": 2.8422427,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 5,
"score": 2.8005157,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 6,
"score": 2.783891,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 2.782668,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 8,
"score": 2.712003,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 9,
"score": 2.498731,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 10,
"score": 2.2379708,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 11,
"score": 2.218436,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 12,
"score": 2.206134,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 2.102532,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 2.0636818,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 15,
"score": 2.0426698,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 16,
"score": 2.0245354,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 17,
"score": 1.9765356,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 1.8800985,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 19,
"score": 1.7948415,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 20,
"score": 1.7783153,
"snippet_found": false
}
],
"expected_ranks": {
"monitoring-and-alerts.md": 1
},
"snippet_found": true,
"passage_rank": 14,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.07142857142857142
}
},
{
"id": "audit-who-changed",
"query": "How can I find out who changed a destination's URL?",
"tags": [
"semantic"
],
"expected_files": [
"audit-log.md"
],
"expected_text": "destination.url_changed",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "invoices-and-payments.md",
"rank": 1,
"score": 4.8638945,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 2,
"score": 4.4864283,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 3,
"score": 4.4840837,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 4.211418,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 5,
"score": 4.071476,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 6,
"score": 4.007015,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 3.974052,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 3.8140574,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 9,
"score": 3.7421665,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 10,
"score": 3.5506918,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 11,
"score": 3.4370897,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 12,
"score": 3.4214826,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 13,
"score": 3.28876,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 3.161325,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 3.1208224,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 16,
"score": 2.7832265,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 2.6834779,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 18,
"score": 2.5932093,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 19,
"score": 2.4568262,
"snippet_found": false
}
],
"expected_ranks": {
"audit-log.md": 3
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "audit-retention",
"query": "How long is the record of admin actions kept?",
"tags": [
"semantic"
],
"expected_files": [
"audit-log.md"
],
"expected_text": "7 years",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "audit-log.md",
"rank": 1,
"score": 6.7694893,
"snippet_found": true
},
{
"filename": "data-retention.md",
"rank": 2,
"score": 5.902327,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 3,
"score": 4.3268433,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 4,
"score": 4.1397276,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 4.104513,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 6,
"score": 4.062165,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 4.053916,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 3.2497015,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 3.210817,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 10,
"score": 3.150314,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 11,
"score": 3.0505908,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 12,
"score": 2.9865234,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 13,
"score": 2.8661394,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 2.650631,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.542187,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 16,
"score": 2.2279477,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 2.0776277,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 18,
"score": 1.9082611,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 19,
"score": 1.9064578,
"snippet_found": false
}
],
"expected_ranks": {
"audit-log.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "audit-tamper",
"query": "Could an admin cover their tracks by deleting log entries?",
"tags": [
"semantic"
],
"expected_files": [
"audit-log.md"
],
"expected_text": "hash",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "roles-and-permissions.md",
"rank": 1,
"score": 5.3748984,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 2,
"score": 5.285913,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 3,
"score": 5.285846,
"snippet_found": true
},
{
"filename": "delivery-retries.md",
"rank": 4,
"score": 4.522437,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.9469934,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 6,
"score": 3.3514473,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 3.1368482,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 8,
"score": 3.126635,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 9,
"score": 2.75564,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 10,
"score": 2.7482202,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 11,
"score": 2.715976,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 12,
"score": 2.5803761,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 13,
"score": 2.460242,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 14,
"score": 2.3662038,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.3606703,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 2.197319,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 17,
"score": 2.1412895,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 18,
"score": 2.0583878,
"snippet_found": false
}
],
"expected_ranks": {
"audit-log.md": 3
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "support-access",
"query": "Can Relaywise staff look at our payloads?",
"tags": [
"semantic"
],
"expected_files": [
"audit-log.md"
],
"expected_text": "Support access",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "audit-log.md",
"rank": 1,
"score": 4.0565033,
"snippet_found": true
},
{
"filename": "transformations.md",
"rank": 2,
"score": 3.8535635,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 3,
"score": 3.7648094,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 4,
"score": 3.761508,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.7174695,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 6,
"score": 3.6284103,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 7,
"score": 3.3209238,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 3.0343482,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 9,
"score": 2.9823442,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 10,
"score": 2.81755,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 2.5708063,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 2.5682056,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 13,
"score": 2.5657067,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 14,
"score": 2.5156288,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 15,
"score": 2.5056129,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 16,
"score": 2.4120002,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 17,
"score": 2.404485,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 18,
"score": 2.3355746,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 19,
"score": 2.2381802,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 20,
"score": 2.1149168,
"snippet_found": false
}
],
"expected_ranks": {
"audit-log.md": 1
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "local-development",
"query": "How do I receive webhooks on my laptop while developing?",
"tags": [
"semantic"
],
"expected_files": [
"cli-reference.md"
],
"expected_text": "rwctl listen",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "cli-reference.md",
"rank": 1,
"score": 5.920323,
"snippet_found": true
},
{
"filename": "monitoring-and-alerts.md",
"rank": 2,
"score": 3.614477,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 3,
"score": 3.504744,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 4,
"score": 3.2636237,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 5,
"score": 3.0580826,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 6,
"score": 3.0453985,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 7,
"score": 2.8457193,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 8,
"score": 2.6562397,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 9,
"score": 2.5232592,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 10,
"score": 2.3091094,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 2.298862,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 12,
"score": 2.2715812,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 13,
"score": 2.1830935,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 14,
"score": 1.984105,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 15,
"score": 1.9079537,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 16,
"score": 1.9055259,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 1.8890364,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 1.8846616,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 1.8736366,
"snippet_found": false
}
],
"expected_ranks": {
"cli-reference.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "cli-in-pipeline",
"query": "How do I authenticate the command-line tool in a CI pipeline?",
"tags": [
"semantic"
],
"expected_files": [
"cli-reference.md"
],
"expected_text": "RELAYWISE_API_KEY",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "cli-reference.md",
"rank": 1,
"score": 6.744873,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 6.4332047,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 3,
"score": 4.639228,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 4.624231,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 4.0451546,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 6,
"score": 3.0904324,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 2.9595504,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 2.9179506,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 2.8892794,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 10,
"score": 2.8273864,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 11,
"score": 2.6481714,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 12,
"score": 2.6116657,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 13,
"score": 2.4750228,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 14,
"score": 2.3840961,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.3526254,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 2.3453667,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 17,
"score": 2.329196,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 2.3105922,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 19,
"score": 2.0088358,
"snippet_found": false
}
],
"expected_ranks": {
"cli-reference.md": 1
},
"snippet_found": true,
"passage_rank": 6,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.16666666666666666
}
},
{
"id": "cli-exit-code-5",
"query": "What does exit code 5 from rwctl mean?",
"tags": [
"exact-token"
],
"expected_files": [
"cli-reference.md"
],
"expected_text": "Rate limited",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "cli-reference.md",
"rank": 1,
"score": 6.5858765,
"snippet_found": true
},
{
"filename": "dead-letter-queue.md",
"rank": 2,
"score": 4.121351,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 3,
"score": 3.7905457,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 4,
"score": 3.750536,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 5,
"score": 3.116803,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 6,
"score": 3.0487542,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 2.8950555,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 8,
"score": 2.7257786,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 9,
"score": 2.7019591,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 10,
"score": 2.624712,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 2.6233444,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 12,
"score": 2.5043364,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 13,
"score": 2.4628139,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 14,
"score": 2.430686,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.3871965,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 2.359127,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 2.3341298,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 18,
"score": 2.2257278,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 19,
"score": 2.1425583,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 20,
"score": 2.125866,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 21,
"score": 2.0921934,
"snippet_found": false
}
],
"expected_ranks": {
"cli-reference.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "client-certificates",
"query": "Our endpoint requires callers to present a client certificate, is that supported?",
"tags": [
"semantic"
],
"expected_files": [
"network-security.md"
],
"expected_text": "mutual TLS",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "network-security.md",
"rank": 1,
"score": 9.485439,
"snippet_found": true
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.8805037,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 3,
"score": 4.1398563,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 4,
"score": 3.9929626,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 5,
"score": 3.6816976,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 6,
"score": 3.5573359,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 7,
"score": 3.1600873,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 3.1363287,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 9,
"score": 3.1233253,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 3.0414858,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 11,
"score": 2.9609323,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 2.8957937,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 2.7488675,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 14,
"score": 2.703147,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 15,
"score": 2.6985838,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 16,
"score": 2.6221676,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 17,
"score": 2.2819054,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 18,
"score": 2.1841009,
"snippet_found": false
}
],
"expected_ranks": {
"network-security.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "restrict-senders",
"query": "Can I restrict which servers are allowed to send events to us?",
"tags": [
"semantic"
],
"expected_files": [
"network-security.md"
],
"expected_text": "ingest allowlist",
"chunks_returned": 50,
"unique_files_returned": 17,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 4.716089,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.5632486,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 4.4465055,
"snippet_found": true
},
{
"filename": "rate-limits.md",
"rank": 4,
"score": 4.4074636,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 5,
"score": 4.3561697,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 6,
"score": 4.2924013,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 7,
"score": 4.2620754,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 8,
"score": 4.0919485,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 9,
"score": 4.0235558,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 10,
"score": 3.9080744,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 3.8798714,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 3.6969318,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 13,
"score": 3.6393707,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 14,
"score": 3.5710607,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 15,
"score": 3.4468415,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 3.0487945,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 17,
"score": 2.857802,
"snippet_found": false
}
],
"expected_ranks": {
"network-security.md": 3
},
"snippet_found": true,
"passage_rank": 3,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.3333333333333333
}
},
{
"id": "enforce-two-step",
"query": "How do I force everyone to use two-step login?",
"tags": [
"semantic"
],
"expected_files": [
"network-security.md"
],
"expected_text": "Require two-factor",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "single-sign-on.md",
"rank": 1,
"score": 4.196766,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.1229253,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 3,
"score": 3.2927356,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 4,
"score": 3.2802827,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 5,
"score": 3.2128882,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 6,
"score": 3.0439441,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 7,
"score": 2.7201033,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 8,
"score": 2.644678,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 9,
"score": 2.5467582,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 10,
"score": 2.535389,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 11,
"score": 2.4999475,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 12,
"score": 2.4157667,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 13,
"score": 2.4001741,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 14,
"score": 2.3948617,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 15,
"score": 2.3878696,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 16,
"score": 2.282506,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.2025251,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 2.0695238,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 19,
"score": 1.9594917,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 20,
"score": 1.9291121,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 21,
"score": 1.796094,
"snippet_found": false
}
],
"expected_ranks": {
"network-security.md": 3
},
"snippet_found": true,
"passage_rank": 9,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.3333333333333333,
"ndcg@1": 0.0,
"ndcg@3": 0.5,
"ndcg@5": 0.5,
"ndcg@10": 0.5,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 1.0,
"passage_rr": 0.1111111111111111
}
},
{
"id": "encryption-at-rest",
"query": "Is stored data encrypted?",
"tags": [
"semantic"
],
"expected_files": [
"network-security.md"
],
"expected_text": "AES-256",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "network-security.md",
"rank": 1,
"score": 4.051931,
"snippet_found": true
},
{
"filename": "data-retention.md",
"rank": 2,
"score": 3.8924236,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 3,
"score": 3.4245567,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 4,
"score": 2.852664,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 2.5582569,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 6,
"score": 2.4678826,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 7,
"score": 2.4120307,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 8,
"score": 2.2670894,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 9,
"score": 2.194287,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 10,
"score": 2.1178303,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 11,
"score": 1.8928216,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 12,
"score": 1.864538,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 13,
"score": 1.8635015,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 14,
"score": 1.7791486,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 15,
"score": 1.5821353,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 16,
"score": 1.5647495,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 17,
"score": 1.5531394,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 1.5409894,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 19,
"score": 1.4597204,
"snippet_found": false
}
],
"expected_ranks": {
"network-security.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "nothing-arriving",
"query": "My endpoint isn't getting any events, where do I start looking?",
"tags": [
"semantic"
],
"expected_files": [
"troubleshooting.md"
],
"expected_text": "Was the event accepted",
"chunks_returned": 50,
"unique_files_returned": 19,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.6265793,
"snippet_found": true
},
{
"filename": "getting-started.md",
"rank": 2,
"score": 4.9406214,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 3,
"score": 4.242301,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 4,
"score": 4.122987,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 5,
"score": 4.0675335,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 6,
"score": 3.9629734,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 7,
"score": 3.9350436,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 8,
"score": 3.9290373,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 9,
"score": 3.8895578,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 10,
"score": 3.573028,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 11,
"score": 3.5166721,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 12,
"score": 3.334948,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 13,
"score": 3.2872982,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 3.243579,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 15,
"score": 3.0642722,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 16,
"score": 3.0577633,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 17,
"score": 3.028301,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 2.8070688,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 19,
"score": 2.774746,
"snippet_found": false
}
],
"expected_ranks": {
"troubleshooting.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "lost-authenticator",
"query": "I lost my authenticator app, how do I get back in?",
"tags": [
"semantic"
],
"expected_files": [
"troubleshooting.md"
],
"expected_text": "second factor",
"chunks_returned": 50,
"unique_files_returned": 21,
"ranked_files": [
{
"filename": "troubleshooting.md",
"rank": 1,
"score": 5.154069,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 2,
"score": 4.039789,
"snippet_found": true
},
{
"filename": "single-sign-on.md",
"rank": 3,
"score": 3.4582567,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 4,
"score": 3.15412,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 5,
"score": 3.0289927,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 3.0060077,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 7,
"score": 2.9968257,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 8,
"score": 2.9520168,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 9,
"score": 2.8144894,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 10,
"score": 2.6343243,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 11,
"score": 2.5093923,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 12,
"score": 2.4954455,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 2.4666529,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 14,
"score": 2.3090854,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 15,
"score": 2.2876267,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 16,
"score": 2.262417,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 17,
"score": 2.1686492,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 18,
"score": 2.117509,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 19,
"score": 2.0579886,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 20,
"score": 1.9386961,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 21,
"score": 1.9223454,
"snippet_found": false
}
],
"expected_ranks": {
"troubleshooting.md": 1
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 1.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
},
{
"id": "first-test-event",
"query": "What is the quickest way to send a first test event?",
"tags": [
"semantic"
],
"expected_files": [
"getting-started.md"
],
"expected_text": "202 Accepted",
"chunks_returned": 50,
"unique_files_returned": 18,
"ranked_files": [
{
"filename": "dead-letter-queue.md",
"rank": 1,
"score": 5.160555,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 2,
"score": 4.864584,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 3,
"score": 4.5930367,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 4,
"score": 4.4245358,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 5,
"score": 4.3252854,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 6,
"score": 4.2423105,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 7,
"score": 4.239122,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 8,
"score": 4.103107,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 9,
"score": 3.8884325,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 10,
"score": 3.7158773,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 11,
"score": 3.655969,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 12,
"score": 3.6182408,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 13,
"score": 3.407936,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 14,
"score": 3.300753,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 15,
"score": 3.2260537,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 16,
"score": 3.218966,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 3.0701206,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 18,
"score": 3.029202,
"snippet_found": false
}
],
"expected_ranks": {
"getting-started.md": 7
},
"snippet_found": false,
"passage_rank": null,
"metrics": {
"recall@1": 0.0,
"recall@3": 0.0,
"recall@5": 0.0,
"recall@10": 0.0,
"reciprocal_rank": 0.0,
"ndcg@1": 0.0,
"ndcg@3": 0.0,
"ndcg@5": 0.0,
"ndcg@10": 0.0,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 0.0,
"passage_hit@10": 0.0,
"passage_rr": 0.0
}
},
{
"id": "source-vs-destination",
"query": "What is the difference between a source and a destination?",
"tags": [
"keyword"
],
"expected_files": [
"getting-started.md"
],
"expected_text": "Where events come from",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "cli-reference.md",
"rank": 1,
"score": 4.9485564,
"snippet_found": false
},
{
"filename": "getting-started.md",
"rank": 2,
"score": 4.9001284,
"snippet_found": true
},
{
"filename": "monitoring-and-alerts.md",
"rank": 3,
"score": 3.985413,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 4,
"score": 3.8785093,
"snippet_found": false
},
{
"filename": "network-security.md",
"rank": 5,
"score": 3.8600652,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 6,
"score": 3.7427316,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 7,
"score": 3.673921,
"snippet_found": false
},
{
"filename": "error-codes.md",
"rank": 8,
"score": 3.6201513,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 9,
"score": 3.4754732,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 10,
"score": 3.3986077,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 11,
"score": 3.2834342,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 12,
"score": 3.2481108,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 13,
"score": 3.2231283,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 14,
"score": 3.2037265,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 15,
"score": 3.149214,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 16,
"score": 3.0159397,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 17,
"score": 2.9234078,
"snippet_found": false
},
{
"filename": "data-retention.md",
"rank": 18,
"score": 2.8746972,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 19,
"score": 2.7781506,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 20,
"score": 2.455618,
"snippet_found": false
}
],
"expected_ranks": {
"getting-started.md": 2
},
"snippet_found": true,
"passage_rank": 4,
"metrics": {
"recall@1": 0.0,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 0.5,
"ndcg@1": 0.0,
"ndcg@3": 0.6309297535714575,
"ndcg@5": 0.6309297535714575,
"ndcg@10": 0.6309297535714575,
"passage_hit@1": 0.0,
"passage_hit@3": 0.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 0.25
}
},
{
"id": "plain-http-destination",
"query": "Can a destination use a plain http URL?",
"tags": [
"keyword",
"multi-doc"
],
"expected_files": [
"getting-started.md",
"error-codes.md"
],
"expected_text": "localhost",
"chunks_returned": 50,
"unique_files_returned": 20,
"ranked_files": [
{
"filename": "getting-started.md",
"rank": 1,
"score": 6.999161,
"snippet_found": true
},
{
"filename": "error-codes.md",
"rank": 2,
"score": 6.419933,
"snippet_found": true
},
{
"filename": "network-security.md",
"rank": 3,
"score": 3.9198084,
"snippet_found": false
},
{
"filename": "cli-reference.md",
"rank": 4,
"score": 3.8565774,
"snippet_found": false
},
{
"filename": "dead-letter-queue.md",
"rank": 5,
"score": 3.5010269,
"snippet_found": false
},
{
"filename": "delivery-retries.md",
"rank": 6,
"score": 3.4314427,
"snippet_found": false
},
{
"filename": "event-filtering.md",
"rank": 7,
"score": 3.1986198,
"snippet_found": false
},
{
"filename": "transformations.md",
"rank": 8,
"score": 3.1823845,
"snippet_found": false
},
{
"filename": "signing-secrets.md",
"rank": 9,
"score": 3.1128054,
"snippet_found": false
},
{
"filename": "audit-log.md",
"rank": 10,
"score": 3.0919387,
"snippet_found": false
},
{
"filename": "configuration-reference.md",
"rank": 11,
"score": 2.918428,
"snippet_found": false
},
{
"filename": "monitoring-and-alerts.md",
"rank": 12,
"score": 2.7586346,
"snippet_found": false
},
{
"filename": "troubleshooting.md",
"rank": 13,
"score": 2.7350254,
"snippet_found": false
},
{
"filename": "regions.md",
"rank": 14,
"score": 2.7063363,
"snippet_found": false
},
{
"filename": "single-sign-on.md",
"rank": 15,
"score": 2.550699,
"snippet_found": false
},
{
"filename": "rate-limits.md",
"rank": 16,
"score": 2.3362465,
"snippet_found": false
},
{
"filename": "invoices-and-payments.md",
"rank": 17,
"score": 2.2635229,
"snippet_found": false
},
{
"filename": "roles-and-permissions.md",
"rank": 18,
"score": 2.2386634,
"snippet_found": false
},
{
"filename": "billing-plans.md",
"rank": 19,
"score": 2.2386358,
"snippet_found": false
},
{
"filename": "api-keys.md",
"rank": 20,
"score": 2.1976867,
"snippet_found": false
}
],
"expected_ranks": {
"getting-started.md": 1,
"error-codes.md": 2
},
"snippet_found": true,
"passage_rank": 1,
"metrics": {
"recall@1": 0.5,
"recall@3": 1.0,
"recall@5": 1.0,
"recall@10": 1.0,
"reciprocal_rank": 1.0,
"ndcg@1": 1.0,
"ndcg@3": 1.0,
"ndcg@5": 1.0,
"ndcg@10": 1.0,
"passage_hit@1": 1.0,
"passage_hit@3": 1.0,
"passage_hit@5": 1.0,
"passage_hit@10": 1.0,
"passage_rr": 1.0
}
}
]
}