Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(mcp): rank every BM25 candidate when post-window filters would hide them
Martin Vogelb95b4033ea1a
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.cindex ad9310186..869df5cc7 100644--- a/src/mcp/mcp.c+++ b/src/mcp/mcp.c@@ -4019,6 +4019,8 @@ enum {* and then joining/filtering/re-ranking those, we bound all work to O(N) where* N = BM25_INNER_LIMIT rather than the full match set size. */BM25_INNER_LIMIT = 2000,+ /* SQLite reads a negative LIMIT as "no limit": rank every candidate. */+ BM25_INNER_ALL = -1,/* Longest label spliced into the MATCH pre-filter; real labels are short* identifiers, so anything longer simply skips the pre-filter. */BM25_LABEL_MAX = 64,@@ -4374,6 +4376,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,return json;}+/* The MATCH expression and the filters the ranked and the count query share.+ * One binder for both keeps them on identical parameters. */+typedef struct {+ const char *fts_query;+ const char *project;+ const char *file_like;+ const char *label;+ const char *kind;+ const char *ns;+ double label_weight;+} bm25_filters_t;++static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {+ if (value && value[0]) {+ sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);+ } else {+ sqlite3_bind_null(st, idx);+ }+}++static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {+ sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);+ sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);+ bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);+ bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);+ bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);+ bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);+ sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);+}++/* The top-candidate window is a performance ceiling, not an exact-total+ * boundary. Probe one candidate beyond it so a broad query never presents a+ * window-local count as the complete match count. The probe shares the+ * label-scoped MATCH but is otherwise global to the FTS table; saturation is+ * therefore conservatively reported even when later project/path filters+ * might discard the hidden candidates. */+static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {+ enum { PROBE_BIND_OFFSET = 2 };+ const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "+ "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";+ sqlite3_stmt *probe = NULL;+ if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {+ return true;+ }+ sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);+ sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);+ bool saturated = sqlite3_step(probe) != SQLITE_DONE;+ sqlite3_finalize(probe);+ return saturated;+}++/* Count the candidates that pass the filters among the top `inner_limit`+ * (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape+ * so FTS5 drives the scan here too. */+static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {+ const char *count_sql =+ "SELECT COUNT(*) FROM ("+ " SELECT fts.rowid FROM ("+ " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"+ " ORDER BY " BM25_WEIGHTS " LIMIT ?3"+ " ) fts "+ " JOIN nodes n ON n.id = fts.rowid "+ " WHERE n.project = ?2 "+ /* MIRRORS the ranked query's filter verbatim — same weights, same+ * label exclusions. Changing one alone reports a total that does+ * not describe the rows returned. */+ " AND " BM25_UNLABELLED_EXCLUSION " "+ " AND (?6 IS NULL OR n.file_path LIKE ?6)"+ " AND (?7 IS NULL OR n.label = ?7)"+ " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "+ " THEN json_extract(n.properties,'$.kind') END) = ?9)"+ " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "+ " THEN json_extract(n.properties,'$.obj_key') END) "+ " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"+ " '_','\\_') || '/%') ESCAPE '\\')"+ ")";+ int total = 0;+ sqlite3_stmt *cs = NULL;+ if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {+ return 0;+ }+ bm25_bind_filters(cs, f);+ sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);+ if (sqlite3_step(cs) == SQLITE_ROW) {+ total = sqlite3_column_int(cs, 0);+ }+ sqlite3_finalize(cs);+ return total;+}+/* Run the BM25 full-text search path and return the JSON result string.* Returns NULL if FTS5 is unavailable or the query produced no usable tokens,* in which case the caller falls back to the regex-based search path. */@@ -4461,117 +4553,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qufree(file_like);return NULL;}- sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);+ const bm25_filters_t filters = {.fts_query = fts_query,+ .project = project,+ .file_like = file_like,+ .label = label,+ .kind = kind,+ .ns = ns,+ .label_weight = label_weight};+ bm25_bind_filters(stmt, &filters);sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);- sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);- if (file_like) {- sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(stmt, BM25_BIND_FILE);- }- if (label && label[0]) {- sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(stmt, BM25_BIND_LABEL);- }sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- if (kind && kind[0]) {- sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(stmt, BM25_BIND_KIND);- }- if (ns && ns[0]) {- sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(stmt, BM25_BIND_NS);- }- sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);-- /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.- * Uses the identical subquery structure so the FTS5 early-exit applies here too. */- int total = 0;- {- const char *count_sql =- "SELECT COUNT(*) FROM ("- " SELECT fts.rowid FROM ("- " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"- " ORDER BY " BM25_WEIGHTS " LIMIT ?3"- " ) fts "- " JOIN nodes n ON n.id = fts.rowid "- " WHERE n.project = ?2 "- /* MIRRORS the ranked query's filter verbatim — same weights, same- * label exclusions. Changing one alone reports a total that does- * not describe the rows returned. */- " AND " BM25_UNLABELLED_EXCLUSION " "- " AND (?6 IS NULL OR n.file_path LIKE ?6)"- " AND (?7 IS NULL OR n.label = ?7)"- " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "- " THEN json_extract(n.properties,'$.kind') END) = ?9)"- " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "- " THEN json_extract(n.properties,'$.obj_key') END) "- " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"- " '_','\\_') || '/%') ESCAPE '\\')"- ")";- sqlite3_stmt *cs = NULL;- if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {- sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);- sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);- if (file_like) {- sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(cs, BM25_BIND_FILE);- }- if (label && label[0]) {- sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(cs, BM25_BIND_LABEL);- }- if (kind && kind[0]) {- sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(cs, BM25_BIND_KIND);- }- if (ns && ns[0]) {- sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);- } else {- sqlite3_bind_null(cs, BM25_BIND_NS);- }- if (sqlite3_step(cs) == SQLITE_ROW) {- total = sqlite3_column_int(cs, 0);- }- sqlite3_finalize(cs);- }- }- /* The top-candidate window is a performance ceiling, not an exact-total- * boundary. Probe one candidate beyond it so a broad query never presents- * a window-local count as the complete match count. The probe shares the- * label-scoped MATCH but is otherwise global to the FTS table; saturation- * is therefore conservatively reported even when later project/path- * filters might discard the hidden candidates. */- bool candidate_window_saturated = true;- {- const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "- "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";- sqlite3_stmt *probe = NULL;- if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {- sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,- MCP_SQLITE_TRANSIENT);- sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);- int probe_step = sqlite3_step(probe);- candidate_window_saturated = probe_step != SQLITE_DONE;- sqlite3_finalize(probe);- }- }+ /* file_pattern, kind and namespace cannot be put into the MATCH the way+ * the label is, so they filter AFTER the candidate window. When more+ * candidates exist beyond it, a narrow filter can find none of its rows in+ * the top BM25_INNER_LIMIT and the search answered 0 rows although+ * matching nodes exist. A saturated window under such a filter -- or one+ * that held no row passing the filters at all -- is therefore ranked over+ * every candidate instead: the FTS-first subquery plan stays, the extra+ * work is linear in the token's match count (the saturation probe already+ * orders every match), and the total becomes exact. The decision depends+ * only on the query, never on the page, so offset pages stay consistent.+ * Unfiltered and label-only queries that find rows keep the window. */+ bool saturated = bm25_window_saturated(db, fts_query);+ bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);+ int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;+ int total = bm25_count_candidates(db, &filters, inner_limit);+ if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {+ inner_limit = BM25_INNER_ALL;+ total = bm25_count_candidates(db, &filters, inner_limit);+ }+ bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;+ sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;bm25_output_row_t *rows =diff --git a/tests/test_mcp.c b/tests/test_mcp.cindex 28d1946b7..2ef16fefe 100644--- a/tests/test_mcp.c+++ b/tests/test_mcp.c@@ -4729,6 +4729,156 @@ TEST(tool_search_graph_bm25_zero_hint_names_the_label) {PASS();}+/* file_pattern, kind and namespace filter after the BM25 candidate window+ * (they cannot be put into the MATCH like the label). 2001 same-label nodes+ * with a short path fill the window ahead of 3 that rank strictly lower (long+ * path) and are the only ones passing each filter, so a windowed search+ * answered 0 rows. A saturated window under such a filter is ranked over+ * every candidate: all 3 rows, an exact total, consistent offset pages. */+static void bm25_post_window_fixture(cbm_store_t *store, const char *project) {+ cbm_store_upsert_project(store, project, "/tmp/bm25-post-window");+ for (int i = 0; i < 2004; i++) {+ bool rare = i >= 2001;+ char qn[64];+ snprintf(qn, sizeof(qn), "bm25-post-window.%s_%04d", rare ? "rare" : "fn", i);+ cbm_node_t node = {+ .project = project,+ .label = "Function",+ .name = "postneedle",+ .qualified_name = qn,+ .file_path = rare ? "deeply/nested/package/layout/registry.c" : "many.c",+ .start_line = i + 1,+ .end_line = i + 1,+ .properties_json =+ rare ? "{\"kind\":\"Deployment\",\"obj_key\":\"apps/Deployment/prod/web\"}" : NULL};+ cbm_store_upsert_node(store, &node);+ }+ cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");+ cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "+ "file_path) SELECT id, cbm_camel_split(name), qualified_name, "+ "label, file_path FROM nodes;");+}++static char *bm25_post_window_search(cbm_mcp_server_t *srv, const char *filter_json, int limit,+ int offset) {+ char req[512];+ snprintf(req, sizeof(req),+ "{\"jsonrpc\":\"2.0\",\"id\":2424,\"method\":\"tools/call\","+ "\"params\":{\"name\":\"search_graph\",\"arguments\":{"+ "\"project\":\"bm25-post-window\",\"query\":\"postneedle\"%s,"+ "\"limit\":%d,\"offset\":%d,\"format\":\"json\"}}}",+ filter_json, limit, offset);+ char *resp = cbm_mcp_server_handle(srv, req);+ char *inner = resp ? extract_text_content(resp) : NULL;+ free(resp);+ return inner;+}++TEST(tool_search_graph_bm25_post_window_filters_rank_every_candidate) {+ cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);+ ASSERT_NOT_NULL(srv);+ cbm_store_t *store = cbm_mcp_server_store(srv);+ ASSERT_NOT_NULL(store);+ cbm_mcp_server_set_project(srv, "bm25-post-window");+ bm25_post_window_fixture(store, "bm25-post-window");++ static const char *const filters[] = {+ ",\"file_pattern\":\"deeply/nested\"",+ ",\"kind\":\"Deployment\"",+ ",\"namespace\":\"prod\"",+ };+ for (size_t i = 0; i < sizeof(filters) / sizeof(filters[0]); i++) {+ char *inner = bm25_post_window_search(srv, filters[i], 10, 0);+ ASSERT_NOT_NULL(inner);+ ASSERT_NOT_NULL(strstr(inner, "\"search_mode\":\"bm25\""));+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-post-window.rare_2001\""));+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-post-window.rare_2003\""));+ ASSERT_NULL(strstr(inner, "\"bm25-post-window.fn_"));+ ASSERT_NOT_NULL(strstr(inner, "\"total\":3,"));+ ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));+ ASSERT_NULL(strstr(inner, "candidate_window_saturated"));+ free(inner);+ }++ /* Offset pages over the exhaustive ranking are disjoint and complete. */+ char *page1 = bm25_post_window_search(srv, filters[0], 2, 0);+ char *page2 = bm25_post_window_search(srv, filters[0], 2, 2);+ ASSERT_NOT_NULL(page1);+ ASSERT_NOT_NULL(page2);+ ASSERT_NOT_NULL(strstr(page1, "\"returned\":2,"));+ ASSERT_NOT_NULL(strstr(page2, "\"returned\":1,"));+ ASSERT_NOT_NULL(strstr(page2, "\"total\":3,"));+ for (int i = 2001; i < 2004; i++) {+ char qn[64];+ snprintf(qn, sizeof(qn), "\"bm25-post-window.rare_%04d\"", i);+ ASSERT_TRUE((strstr(page1, qn) != NULL) != (strstr(page2, qn) != NULL));+ }+ free(page1);+ free(page2);++ /* Without a post-window filter the window stays and says so. */+ char *inner = bm25_post_window_search(srv, "", 10, 0);+ ASSERT_NOT_NULL(inner);+ ASSERT_NOT_NULL(strstr(inner, "\"total\":2000,"));+ ASSERT_NOT_NULL(strstr(inner, "\"candidate_window_saturated\":true"));+ free(inner);++ cbm_mcp_server_free(srv);+ PASS();+}++/* An unlabelled query whose whole candidate window is File/Folder/Variable/+ * Project rows (excluded from unlabelled ranking) found no row at all, while+ * a matching definition sat just beyond the window: 2001 short Folder rows+ * outrank one Function with a long path. A window with no row passing the+ * filters is ranked over every candidate instead. */+TEST(tool_search_graph_bm25_empty_window_ranks_every_candidate) {+ cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);+ ASSERT_NOT_NULL(srv);+ cbm_store_t *store = cbm_mcp_server_store(srv);+ ASSERT_NOT_NULL(store);+ const char *project = "bm25-empty-window";+ cbm_mcp_server_set_project(srv, project);+ ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/bm25-empty-window"), CBM_STORE_OK);+ for (int i = 0; i < 2002; i++) {+ bool def = i == 2001;+ char qn[64];+ snprintf(qn, sizeof(qn), "bm25-empty-window.%s_%04d", def ? "def" : "dir", i);+ cbm_node_t node = {.project = project,+ .label = def ? "Function" : "Folder",+ .name = "foldneedle",+ .qualified_name = qn,+ .file_path = def ? "deeply/nested/package/layout/registry.c" : "d",+ .start_line = i + 1,+ .end_line = i + 1};+ ASSERT_GT(cbm_store_upsert_node(store, &node), 0);+ }+ ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');"),+ CBM_STORE_OK);+ ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "+ "file_path) SELECT id, cbm_camel_split(name), qualified_name, "+ "label, file_path FROM nodes;"),+ CBM_STORE_OK);++ char *resp = cbm_mcp_server_handle(+ srv, "{\"jsonrpc\":\"2.0\",\"id\":2425,\"method\":\"tools/call\","+ "\"params\":{\"name\":\"search_graph\",\"arguments\":{"+ "\"project\":\"bm25-empty-window\",\"query\":\"foldneedle\",\"limit\":10,"+ "\"format\":\"json\"}}}");+ ASSERT_NOT_NULL(resp);+ char *inner = extract_text_content(resp);+ free(resp);+ ASSERT_NOT_NULL(inner);+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-empty-window.def_2001\",\"Function\""));+ ASSERT_NULL(strstr(inner, "\"Folder\""));+ ASSERT_NOT_NULL(strstr(inner, "\"total\":1,"));+ ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));+ free(inner);++ cbm_mcp_server_free(srv);+ PASS();+}+/* The definition whose NAME is the query ranks first: the `Table` class above* its own methods and above test methods that repeat "table" three times; the* `get_object_or_404` function above the test methods that contain it. */@@ -25336,6 +25486,8 @@ SUITE(mcp) {RUN_TEST(tool_search_graph_bm25_label_scope_keeps_rank_order);RUN_TEST(tool_search_graph_bm25_serves_unranked_labels_when_named);RUN_TEST(tool_search_graph_bm25_zero_hint_names_the_label);+ RUN_TEST(tool_search_graph_bm25_post_window_filters_rank_every_candidate);+ RUN_TEST(tool_search_graph_bm25_empty_window_ranks_every_candidate);RUN_TEST(tool_search_graph_bm25_ranks_exact_name_first);RUN_TEST(tool_search_graph_rejects_bm25_and_semantic_query_together);RUN_TEST(tool_search_graph_semantic_ceiling_never_emits_unusable_continuation);