Code graph and memory for your coding agents, in one Rust program. For Claude Code, Codex, Cursor and any MCP client.
Rust agent-memoryai-agentsclaude-codecode-analysiscode-graphcode-intelligencecode-searchcodexcoding-agentscontext-engineeringcursordeveloper-toolsgraphragknowledge-graphlocal-firstmcpmcp-servermodel-context-protocolrusttree-sitter
{
"about": "Graphicle's offline retrieval benchmark (docs/benchmarks/retrieval.md): every issue of both sets with its repository, language and number of gold files, each system's results (hit@k: the issues with a gold file among the first k files; MRR) and, for each pair of systems compared, the issues where only one of the two has a hit at each k. retrieval_chart.py draws docs/assets/retrieval-*.svg and .png and prints the page's tables from this file.",
"measured": {
"2026-10-21": "Graphicle 0.1.3 and 0.1.2, BM25 and grep, both sets",
"2026-10-25": "Graphicle 0.1.8, held-out set"
},
"systems": {
"graphicle-0.1.8": {
"label": "Graphicle 0.1.8",
"legend": "Graphicle 0.1.8",
"short": "0.1.8",
"color": "graphicle"
},
"graphicle-0.1.8-ranked": {
"label": "Graphicle 0.1.8, its ranking before the packet",
"legend": "",
"short": ""
},
"graphicle-0.1.3": {
"label": "Graphicle 0.1.3",
"legend": "Graphicle 0.1.3",
"short": "0.1.3",
"color": "graphicle-earlier"
},
"graphicle-0.1.2": {
"label": "Graphicle 0.1.2",
"legend": "Graphicle 0.1.2",
"short": "0.1.2",
"color": "graphicle-earlier"
},
"bm25": {
"label": "BM25, SWE-bench style",
"legend": "BM25 (SWE-bench style)",
"short": "BM25",
"color": "baseline"
},
"grep": {
"label": "grep for the names in the issue (rg 14.1.1)",
"legend": "grep for the issue's names",
"short": "grep",
"color": "baseline-2"
},
"graphicle-0.1.3-ranked": {
"label": "Graphicle 0.1.3, its ranking before the packet",
"legend": "",
"short": ""
},
"graphicle-0.1.2-ranked": {
"label": "Graphicle 0.1.2, its ranking before the packet",
"legend": "",
"short": ""
}
},
"sets": {
"held-out": {
"table_title": "Held-out set: SWE-bench Verified, never used for tuning",
"tasks": [
{
"id": "astropy__astropy-13453",
"repo": "astropy/astropy",
"language": "Python",
"gold_files": 1
},
{
"id": "astropy__astropy-13579",
"repo": "astropy/astropy",
"language": "Python",
"gold_files": 1
},
{
"id": "astropy__astropy-7336",
"repo": "astropy/astropy",
"language": "Python",
"gold_files": 1
},
{
"id": "astropy__astropy-8872",
"repo": "astropy/astropy",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-10914",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-11163",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-11239",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-11400",
"repo": "django/django",
"language": "Python",
"gold_files": 3
},
{
"id": "django__django-11477",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-11790",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-12155",
"repo": "django/django",
"language": "Python",
"gold_files": 2
},
{
"id": "django__django-12273",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-12406",
"repo": "django/django",
"language": "Python",
"gold_files": 2
},
{
"id": "django__django-12774",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-12965",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13158",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13195",
"repo": "django/django",
"language": "Python",
"gold_files": 3
},
{
"id": "django__django-13212",
"repo": "django/django",
"language": "Python",
"gold_files": 2
},
{
"id": "django__django-13315",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13343",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13406",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13410",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13551",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13741",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13810",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13821",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13837",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13933",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-13964",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14155",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14349",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14373",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14493",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14580",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14765",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14855",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-14999",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-15252",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-15277",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-15382",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-15629",
"repo": "django/django",
"language": "Python",
"gold_files": 4
},
{
"id": "django__django-15695",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16082",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16315",
"repo": "django/django",
"language": "Python",
"gold_files": 2
},
{
"id": "django__django-16333",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16485",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16493",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16527",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-16819",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "django__django-9296",
"repo": "django/django",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-20676",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-23412",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-23476",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-24570",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-25287",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-26113",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "matplotlib__matplotlib-26466",
"repo": "matplotlib/matplotlib",
"language": "Python",
"gold_files": 1
},
{
"id": "psf__requests-1142",
"repo": "psf/requests",
"language": "Python",
"gold_files": 1
},
{
"id": "psf__requests-5414",
"repo": "psf/requests",
"language": "Python",
"gold_files": 1
},
{
"id": "pydata__xarray-3095",
"repo": "pydata/xarray",
"language": "Python",
"gold_files": 2
},
{
"id": "pydata__xarray-3677",
"repo": "pydata/xarray",
"language": "Python",
"gold_files": 1
},
{
"id": "pydata__xarray-4356",
"repo": "pydata/xarray",
"language": "Python",
"gold_files": 1
},
{
"id": "pydata__xarray-4695",
"repo": "pydata/xarray",
"language": "Python",
"gold_files": 1
},
{
"id": "pylint-dev__pylint-4661",
"repo": "pylint-dev/pylint",
"language": "Python",
"gold_files": 1
},
{
"id": "pylint-dev__pylint-6528",
"repo": "pylint-dev/pylint",
"language": "Python",
"gold_files": 2
},
{
"id": "pytest-dev__pytest-10051",
"repo": "pytest-dev/pytest",
"language": "Python",
"gold_files": 1
},
{
"id": "pytest-dev__pytest-10081",
"repo": "pytest-dev/pytest",
"language": "Python",
"gold_files": 1
},
{
"id": "pytest-dev__pytest-5809",
"repo": "pytest-dev/pytest",
"language": "Python",
"gold_files": 1
},
{
"id": "pytest-dev__pytest-6202",
"repo": "pytest-dev/pytest",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-10297",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-10908",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-13124",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-13328",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-25747",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-25931",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "scikit-learn__scikit-learn-9288",
"repo": "scikit-learn/scikit-learn",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-10323",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-11445",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-11510",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-7985",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-8459",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-8721",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-9229",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-9281",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sphinx-doc__sphinx-9602",
"repo": "sphinx-doc/sphinx",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-11618",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-12096",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-12419",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-13091",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 21
},
{
"id": "sympy__sympy-14248",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 3
},
{
"id": "sympy__sympy-14976",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-15875",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-16450",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-17630",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-18211",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-19637",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-19783",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 2
},
{
"id": "sympy__sympy-22080",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 2
},
{
"id": "sympy__sympy-24213",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
},
{
"id": "sympy__sympy-24562",
"repo": "sympy/sympy",
"language": "Python",
"gold_files": 1
}
],
"n": 100,
"results": {
"graphicle-0.1.8": {
"hit@1": 43,
"hit@5": 75,
"hit@10": 81,
"mrr": 0.5606
},
"graphicle-0.1.8-ranked": {
"hit@1": 42,
"hit@5": 76,
"hit@10": 81,
"mrr": 0.5632
},
"graphicle-0.1.3": {
"hit@1": 43,
"hit@5": 75,
"hit@10": 82,
"mrr": 0.5617
},
"graphicle-0.1.2": {
"hit@1": 13,
"hit@5": 30,
"hit@10": 30,
"mrr": 0.1912
},
"bm25": {
"hit@1": 35,
"hit@5": 61,
"hit@10": 77,
"mrr": 0.4776
},
"grep": {
"hit@1": 20,
"hit@5": 54,
"hit@10": 71,
"mrr": 0.3653
},
"graphicle-0.1.3-ranked": {
"hit@1": 42,
"hit@5": 76,
"hit@10": 82,
"mrr": 0.5639
},
"graphicle-0.1.2-ranked": {
"hit@1": 16,
"hit@5": 29,
"hit@10": 35,
"mrr": 0.2251
}
},
"mcnemar": [
{
"a": "graphicle-0.1.8",
"b": "bm25",
"1": {
"only_a": 19,
"only_b": 11
},
"5": {
"only_a": 21,
"only_b": 7
},
"10": {
"only_a": 13,
"only_b": 9
}
},
{
"a": "graphicle-0.1.3",
"b": "bm25",
"1": {
"only_a": 19,
"only_b": 11
},
"5": {
"only_a": 21,
"only_b": 7
},
"10": {
"only_a": 13,
"only_b": 8
}
}
],
"table_order": [
"graphicle-0.1.8",
"bm25",
"grep",
"graphicle-0.1.8-ranked",
"graphicle-0.1.3",
"graphicle-0.1.2",
"graphicle-0.1.3-ranked",
"graphicle-0.1.2-ranked"
]
},
"dev": {
"table_title": "Development set: issues filed in July 2026, used for tuning",
"tasks": [
{
"id": "DynamoRIO__dynamorio-8017",
"repo": "DynamoRIO/dynamorio",
"language": "C",
"gold_files": 3
},
{
"id": "IfcOpenShell__IfcOpenShell-8887",
"repo": "IfcOpenShell/IfcOpenShell",
"language": "C++",
"gold_files": 1
},
{
"id": "JasperFx__marten-5080",
"repo": "JasperFx/marten",
"language": "C#",
"gold_files": 1
},
{
"id": "JasperFx__wolverine-3456",
"repo": "JasperFx/wolverine",
"language": "C#",
"gold_files": 5
},
{
"id": "JasperFx__wolverine-3582",
"repo": "JasperFx/wolverine",
"language": "C#",
"gold_files": 1
},
{
"id": "JasperFx__wolverine-3695",
"repo": "JasperFx/wolverine",
"language": "C#",
"gold_files": 3
},
{
"id": "SubtitleEdit__subtitleedit-12151",
"repo": "SubtitleEdit/subtitleedit",
"language": "C#",
"gold_files": 2
},
{
"id": "SubtitleEdit__subtitleedit-12763",
"repo": "SubtitleEdit/subtitleedit",
"language": "C#",
"gold_files": 1
},
{
"id": "SubtitleEdit__subtitleedit-12895",
"repo": "SubtitleEdit/subtitleedit",
"language": "C#",
"gold_files": 1
},
{
"id": "agentgateway__agentgateway-2452",
"repo": "agentgateway/agentgateway",
"language": "Rust",
"gold_files": 1
},
{
"id": "agentscope-ai__agentscope-java-2189",
"repo": "agentscope-ai/agentscope-java",
"language": "Java",
"gold_files": 2
},
{
"id": "agentscope-ai__agentscope-java-2257",
"repo": "agentscope-ai/agentscope-java",
"language": "Java",
"gold_files": 1
},
{
"id": "agentscope-ai__agentscope-java-2260",
"repo": "agentscope-ai/agentscope-java",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__paimon-8536",
"repo": "apache/paimon",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__paimon-8874",
"repo": "apache/paimon",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__parquet-java-3642",
"repo": "apache/parquet-java",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__parquet-java-3645",
"repo": "apache/parquet-java",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__parquet-java-3669",
"repo": "apache/parquet-java",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__shardingsphere-39104",
"repo": "apache/shardingsphere",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__shardingsphere-39137",
"repo": "apache/shardingsphere",
"language": "Java",
"gold_files": 1
},
{
"id": "apache__shardingsphere-39237",
"repo": "apache/shardingsphere",
"language": "Java",
"gold_files": 1
},
{
"id": "btraceio__btrace-894",
"repo": "btraceio/btrace",
"language": "Java",
"gold_files": 3
},
{
"id": "btraceio__btrace-895",
"repo": "btraceio/btrace",
"language": "Java",
"gold_files": 5
},
{
"id": "btraceio__btrace-910",
"repo": "btraceio/btrace",
"language": "Java",
"gold_files": 4
},
{
"id": "c-ares__c-ares-1190",
"repo": "c-ares/c-ares",
"language": "C",
"gold_files": 4
},
{
"id": "colbymchenry__codegraph-1317",
"repo": "colbymchenry/codegraph",
"language": "C",
"gold_files": 2
},
{
"id": "colbymchenry__codegraph-1318",
"repo": "colbymchenry/codegraph",
"language": "C",
"gold_files": 1
},
{
"id": "colbymchenry__codegraph-1370",
"repo": "colbymchenry/codegraph",
"language": "C",
"gold_files": 1
},
{
"id": "duckdb__ducklake-1339",
"repo": "duckdb/ducklake",
"language": "C++",
"gold_files": 2
},
{
"id": "duckdb__ducklake-1340",
"repo": "duckdb/ducklake",
"language": "C++",
"gold_files": 3
},
{
"id": "elsa-workflows__elsa-core-7898",
"repo": "elsa-workflows/elsa-core",
"language": "C#",
"gold_files": 1
},
{
"id": "floci-io__floci-1837",
"repo": "floci-io/floci",
"language": "Java",
"gold_files": 1
},
{
"id": "floci-io__floci-1870",
"repo": "floci-io/floci",
"language": "Java",
"gold_files": 1
},
{
"id": "floci-io__floci-2015",
"repo": "floci-io/floci",
"language": "Java",
"gold_files": 2
},
{
"id": "fluent__fluent-bit-12029",
"repo": "fluent/fluent-bit",
"language": "C",
"gold_files": 2
},
{
"id": "gitbutlerapp__gitbutler-14920",
"repo": "gitbutlerapp/gitbutler",
"language": "Rust",
"gold_files": 2
},
{
"id": "gleam-lang__gleam-5958",
"repo": "gleam-lang/gleam",
"language": "Rust",
"gold_files": 1
},
{
"id": "gleam-lang__gleam-6021",
"repo": "gleam-lang/gleam",
"language": "Rust",
"gold_files": 3
},
{
"id": "h4ckf0r0day__obscura-412",
"repo": "h4ckf0r0day/obscura",
"language": "Rust",
"gold_files": 1
},
{
"id": "h4ckf0r0day__obscura-421",
"repo": "h4ckf0r0day/obscura",
"language": "Rust",
"gold_files": 1
},
{
"id": "h4ckf0r0day__obscura-440",
"repo": "h4ckf0r0day/obscura",
"language": "Rust",
"gold_files": 1
},
{
"id": "jsonata-js__jsonata-814",
"repo": "jsonata-js/jsonata",
"language": "JS/TS",
"gold_files": 4
},
{
"id": "kvcache-ai__Mooncake-2892",
"repo": "kvcache-ai/Mooncake",
"language": "C++",
"gold_files": 1
},
{
"id": "kvcache-ai__Mooncake-2964",
"repo": "kvcache-ai/Mooncake",
"language": "C++",
"gold_files": 2
},
{
"id": "lakehq__sail-2292",
"repo": "lakehq/sail",
"language": "Rust",
"gold_files": 1
},
{
"id": "lance-format__lance-7618",
"repo": "lance-format/lance",
"language": "Rust",
"gold_files": 1
},
{
"id": "lance-format__lance-8049",
"repo": "lance-format/lance",
"language": "Rust",
"gold_files": 1
},
{
"id": "langchain4j__langchain4j-5755",
"repo": "langchain4j/langchain4j",
"language": "Java",
"gold_files": 1
},
{
"id": "langchain4j__langchain4j-5847",
"repo": "langchain4j/langchain4j",
"language": "Java",
"gold_files": 2
},
{
"id": "langchain4j__langchain4j-5869",
"repo": "langchain4j/langchain4j",
"language": "Java",
"gold_files": 2
},
{
"id": "luau-lang__luau-2506",
"repo": "luau-lang/luau",
"language": "C++",
"gold_files": 1
},
{
"id": "modelcontextprotocol__rust-sdk-1005",
"repo": "modelcontextprotocol/rust-sdk",
"language": "Rust",
"gold_files": 1
},
{
"id": "modelcontextprotocol__rust-sdk-1021",
"repo": "modelcontextprotocol/rust-sdk",
"language": "Rust",
"gold_files": 4
},
{
"id": "modelcontextprotocol__rust-sdk-996",
"repo": "modelcontextprotocol/rust-sdk",
"language": "Rust",
"gold_files": 1
},
{
"id": "nearai__ironclaw-6706",
"repo": "nearai/ironclaw",
"language": "Rust",
"gold_files": 4
},
{
"id": "nearai__ironclaw-6712",
"repo": "nearai/ironclaw",
"language": "Rust",
"gold_files": 2
},
{
"id": "notionnext-org__NotionNext-4290",
"repo": "notionnext-org/NotionNext",
"language": "JS/TS",
"gold_files": 1
},
{
"id": "nunit__nunit-5351",
"repo": "nunit/nunit",
"language": "C#",
"gold_files": 3
},
{
"id": "open-gsd__gsd-core-2079",
"repo": "open-gsd/gsd-core",
"language": "JS/TS",
"gold_files": 1
},
{
"id": "open-gsd__gsd-core-2233",
"repo": "open-gsd/gsd-core",
"language": "JS/TS",
"gold_files": 1
},
{
"id": "open-gsd__gsd-core-2261",
"repo": "open-gsd/gsd-core",
"language": "JS/TS",
"gold_files": 2
},
{
"id": "playcanvas__engine-9000",
"repo": "playcanvas/engine",
"language": "JS/TS",
"gold_files": 1
},
{
"id": "playcanvas__engine-9002",
"repo": "playcanvas/engine",
"language": "JS/TS",
"gold_files": 2
},
{
"id": "playcanvas__engine-9066",
"repo": "playcanvas/engine",
"language": "JS/TS",
"gold_files": 2
},
{
"id": "prefix-dev__pixi-6518",
"repo": "prefix-dev/pixi",
"language": "Rust",
"gold_files": 4
},
{
"id": "prefix-dev__pixi-6529",
"repo": "prefix-dev/pixi",
"language": "Rust",
"gold_files": 4
},
{
"id": "root-project__root-22854",
"repo": "root-project/root",
"language": "C++",
"gold_files": 1
},
{
"id": "root-project__root-22857",
"repo": "root-project/root",
"language": "C++",
"gold_files": 1
},
{
"id": "rust-lang__rust-analyzer-22751",
"repo": "rust-lang/rust-analyzer",
"language": "Rust",
"gold_files": 1
},
{
"id": "rust-lang__rust-analyzer-22804",
"repo": "rust-lang/rust-analyzer",
"language": "Rust",
"gold_files": 1
},
{
"id": "rust-lang__rust-analyzer-22855",
"repo": "rust-lang/rust-analyzer",
"language": "Rust",
"gold_files": 1
},
{
"id": "web-infra-dev__rspack-14794",
"repo": "web-infra-dev/rspack",
"language": "Rust",
"gold_files": 1
},
{
"id": "web-infra-dev__rspack-14803",
"repo": "web-infra-dev/rspack",
"language": "Rust",
"gold_files": 1
},
{
"id": "web-infra-dev__rspack-14882",
"repo": "web-infra-dev/rspack",
"language": "Rust",
"gold_files": 1
},
{
"id": "xroche__httrack-599",
"repo": "xroche/httrack",
"language": "C",
"gold_files": 1
},
{
"id": "xroche__httrack-618",
"repo": "xroche/httrack",
"language": "C",
"gold_files": 1
},
{
"id": "xroche__httrack-811",
"repo": "xroche/httrack",
"language": "C",
"gold_files": 1
},
{
"id": "yvgude__lean-ctx-1179",
"repo": "yvgude/lean-ctx",
"language": "Rust",
"gold_files": 1
},
{
"id": "yvgude__lean-ctx-816",
"repo": "yvgude/lean-ctx",
"language": "Rust",
"gold_files": 4
},
{
"id": "yvgude__lean-ctx-969",
"repo": "yvgude/lean-ctx",
"language": "Rust",
"gold_files": 1
}
],
"n": 80,
"results": {
"graphicle-0.1.3": {
"hit@1": 38,
"hit@5": 63,
"hit@10": 69,
"mrr": 0.6112
},
"graphicle-0.1.2": {
"hit@1": 12,
"hit@5": 27,
"hit@10": 27,
"mrr": 0.2269
},
"bm25": {
"hit@1": 34,
"hit@5": 51,
"hit@10": 57,
"mrr": 0.5225
},
"grep": {
"hit@1": 30,
"hit@5": 51,
"hit@10": 59,
"mrr": 0.5014
},
"graphicle-0.1.3-ranked": {
"hit@1": 37,
"hit@5": 63,
"hit@10": 69,
"mrr": 0.6024
},
"graphicle-0.1.2-ranked": {
"hit@1": 8,
"hit@5": 26,
"hit@10": 29,
"mrr": 0.2133
}
},
"mcnemar": [
{
"a": "graphicle-0.1.3",
"b": "bm25",
"1": {
"only_a": 11,
"only_b": 7
},
"5": {
"only_a": 16,
"only_b": 4
},
"10": {
"only_a": 14,
"only_b": 2
}
}
],
"table_order": [
"graphicle-0.1.3",
"graphicle-0.1.2",
"bm25",
"grep",
"graphicle-0.1.3-ranked",
"graphicle-0.1.2-ranked"
]
}
},
"chart": {
"set": "held-out",
"systems": [
"graphicle-0.1.8",
"bm25",
"grep"
],
"compare": [
"graphicle-0.1.8",
"bm25"
],
"tagline": "Offline retrieval benchmark, 25 October 2026",
"title": "Finding the files a fix changes: 100 SWE-bench Verified issues, held out from tuning",
"subtitle": "For each issue: is a file the fix changed among the first k files? Graphicle: the files its context packet cites, then lists. No model calls. Higher is better.",
"panels": {
"hit@1": {
"title": "Top 1",
"sub": "Issues of {n} with a changed file first"
},
"hit@5": {
"title": "Top 5",
"sub": "Issues of {n} with a changed file in the first 5"
},
"hit@10": {
"title": "Top 10",
"sub": "Issues of {n} with a changed file in the first 10"
},
"mrr": {
"title": "MRR",
"sub": "Mean reciprocal rank of the first changed file",
"note": "1 when it comes first, 0.5 when second, 0 past rank 100."
}
},
"compare_label": "0.1.8 vs BM25, McNemar exact:",
"footer": [
"Graphicle 0.1.8, a release build, runs graphicle context on the issue's prompt with its defaults. BM25: SWE-bench style, one document per source file. grep: rg 14.1.1 for the identifiers and paths the issue names, files ranked by names matched. BM25 and grep as measured on 21 October 2026, on the same checkouts.",
"This measures ranking, not whether a coding agent fixes more issues. No agent test of this ranking has been run yet."
],
"url_label": "Method, task list and limits:",
"url": "github.com/graphicle-ai/graphicle/blob/main/docs/benchmarks/retrieval.md"
}
}