Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(mcp): rank every BM25 candidate when post-window filters would hide them

Martin Vogelb95b4033ea1a
diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c
index ad9310186..869df5cc7 100644
--- a/src/mcp/mcp.c
+++ b/src/mcp/mcp.c
@@ -4019,6 +4019,8 @@ enum {
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
* N = BM25_INNER_LIMIT rather than the full match set size. */
BM25_INNER_LIMIT = 2000,
+ /* SQLite reads a negative LIMIT as "no limit": rank every candidate. */
+ BM25_INNER_ALL = -1,
/* Longest label spliced into the MATCH pre-filter; real labels are short
* identifiers, so anything longer simply skips the pre-filter. */
BM25_LABEL_MAX = 64,
@@ -4374,6 +4376,96 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
return json;
}
+/* The MATCH expression and the filters the ranked and the count query share.
+ * One binder for both keeps them on identical parameters. */
+typedef struct {
+ const char *fts_query;
+ const char *project;
+ const char *file_like;
+ const char *label;
+ const char *kind;
+ const char *ns;
+ double label_weight;
+} bm25_filters_t;
+
+static void bm25_bind_opt_text(sqlite3_stmt *st, int idx, const char *value) {
+ if (value && value[0]) {
+ sqlite3_bind_text(st, idx, value, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
+ } else {
+ sqlite3_bind_null(st, idx);
+ }
+}
+
+static void bm25_bind_filters(sqlite3_stmt *st, const bm25_filters_t *f) {
+ sqlite3_bind_text(st, BM25_BIND_QUERY, f->fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
+ sqlite3_bind_text(st, BM25_BIND_PROJECT, f->project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
+ bm25_bind_opt_text(st, BM25_BIND_FILE, f->file_like);
+ bm25_bind_opt_text(st, BM25_BIND_LABEL, f->label);
+ bm25_bind_opt_text(st, BM25_BIND_KIND, f->kind);
+ bm25_bind_opt_text(st, BM25_BIND_NS, f->ns);
+ sqlite3_bind_double(st, BM25_BIND_LABEL_WEIGHT, f->label_weight);
+}
+
+/* The top-candidate window is a performance ceiling, not an exact-total
+ * boundary. Probe one candidate beyond it so a broad query never presents a
+ * window-local count as the complete match count. The probe shares the
+ * label-scoped MATCH but is otherwise global to the FTS table; saturation is
+ * therefore conservatively reported even when later project/path filters
+ * might discard the hidden candidates. */
+static bool bm25_window_saturated(sqlite3 *db, const char *fts_query) {
+ enum { PROBE_BIND_OFFSET = 2 };
+ const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
+ "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
+ sqlite3_stmt *probe = NULL;
+ if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) != SQLITE_OK) {
+ return true;
+ }
+ sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
+ sqlite3_bind_int(probe, PROBE_BIND_OFFSET, BM25_INNER_LIMIT);
+ bool saturated = sqlite3_step(probe) != SQLITE_DONE;
+ sqlite3_finalize(probe);
+ return saturated;
+}
+
+/* Count the candidates that pass the filters among the top `inner_limit`
+ * (BM25_INNER_ALL: among all of them), with the ranked query's subquery shape
+ * so FTS5 drives the scan here too. */
+static int bm25_count_candidates(sqlite3 *db, const bm25_filters_t *f, int inner_limit) {
+ const char *count_sql =
+ "SELECT COUNT(*) FROM ("
+ " SELECT fts.rowid FROM ("
+ " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
+ " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
+ " ) fts "
+ " JOIN nodes n ON n.id = fts.rowid "
+ " WHERE n.project = ?2 "
+ /* MIRRORS the ranked query's filter verbatim — same weights, same
+ * label exclusions. Changing one alone reports a total that does
+ * not describe the rows returned. */
+ " AND " BM25_UNLABELLED_EXCLUSION " "
+ " AND (?6 IS NULL OR n.file_path LIKE ?6)"
+ " AND (?7 IS NULL OR n.label = ?7)"
+ " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
+ " THEN json_extract(n.properties,'$.kind') END) = ?9)"
+ " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
+ " THEN json_extract(n.properties,'$.obj_key') END) "
+ " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
+ " '_','\\_') || '/%') ESCAPE '\\')"
+ ")";
+ int total = 0;
+ sqlite3_stmt *cs = NULL;
+ if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) != SQLITE_OK) {
+ return 0;
+ }
+ bm25_bind_filters(cs, f);
+ sqlite3_bind_int(cs, BM25_BIND_LIMIT, inner_limit);
+ if (sqlite3_step(cs) == SQLITE_ROW) {
+ total = sqlite3_column_int(cs, 0);
+ }
+ sqlite3_finalize(cs);
+ return total;
+}
+
/* Run the BM25 full-text search path and return the JSON result string.
* Returns NULL if FTS5 is unavailable or the query produced no usable tokens,
* in which case the caller falls back to the regex-based search path. */
@@ -4461,117 +4553,39 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
free(file_like);
return NULL;
}
- sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
+ const bm25_filters_t filters = {.fts_query = fts_query,
+ .project = project,
+ .file_like = file_like,
+ .label = label,
+ .kind = kind,
+ .ns = ns,
+ .label_weight = label_weight};
+ bm25_bind_filters(stmt, &filters);
sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT);
sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0);
- sqlite3_bind_int(stmt, BM25_BIND_INNER, BM25_INNER_LIMIT);
- if (file_like) {
- sqlite3_bind_text(stmt, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(stmt, BM25_BIND_FILE);
- }
- if (label && label[0]) {
- sqlite3_bind_text(stmt, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(stmt, BM25_BIND_LABEL);
- }
sqlite3_bind_text(stmt, BM25_BIND_EXACT, query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- if (kind && kind[0]) {
- sqlite3_bind_text(stmt, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(stmt, BM25_BIND_KIND);
- }
- if (ns && ns[0]) {
- sqlite3_bind_text(stmt, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(stmt, BM25_BIND_NS);
- }
- sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
-
- /* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
- * Uses the identical subquery structure so the FTS5 early-exit applies here too. */
- int total = 0;
- {
- const char *count_sql =
- "SELECT COUNT(*) FROM ("
- " SELECT fts.rowid FROM ("
- " SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1"
- " ORDER BY " BM25_WEIGHTS " LIMIT ?3"
- " ) fts "
- " JOIN nodes n ON n.id = fts.rowid "
- " WHERE n.project = ?2 "
- /* MIRRORS the ranked query's filter verbatim — same weights, same
- * label exclusions. Changing one alone reports a total that does
- * not describe the rows returned. */
- " AND " BM25_UNLABELLED_EXCLUSION " "
- " AND (?6 IS NULL OR n.file_path LIKE ?6)"
- " AND (?7 IS NULL OR n.label = ?7)"
- " AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
- " THEN json_extract(n.properties,'$.kind') END) = ?9)"
- " AND (?10 IS NULL OR (CASE WHEN json_valid(n.properties) "
- " THEN json_extract(n.properties,'$.obj_key') END) "
- " LIKE ('%/%/' || replace(replace(replace(?10,'\\','\\\\'),'%','\\%'),"
- " '_','\\_') || '/%') ESCAPE '\\')"
- ")";
- sqlite3_stmt *cs = NULL;
- if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) {
- sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
- sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
- if (file_like) {
- sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(cs, BM25_BIND_FILE);
- }
- if (label && label[0]) {
- sqlite3_bind_text(cs, BM25_BIND_LABEL, label, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(cs, BM25_BIND_LABEL);
- }
- if (kind && kind[0]) {
- sqlite3_bind_text(cs, BM25_BIND_KIND, kind, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(cs, BM25_BIND_KIND);
- }
- if (ns && ns[0]) {
- sqlite3_bind_text(cs, BM25_BIND_NS, ns, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
- } else {
- sqlite3_bind_null(cs, BM25_BIND_NS);
- }
- if (sqlite3_step(cs) == SQLITE_ROW) {
- total = sqlite3_column_int(cs, 0);
- }
- sqlite3_finalize(cs);
- }
- }
- /* The top-candidate window is a performance ceiling, not an exact-total
- * boundary. Probe one candidate beyond it so a broad query never presents
- * a window-local count as the complete match count. The probe shares the
- * label-scoped MATCH but is otherwise global to the FTS table; saturation
- * is therefore conservatively reported even when later project/path
- * filters might discard the hidden candidates. */
- bool candidate_window_saturated = true;
- {
- const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
- "ORDER BY bm25(nodes_fts), rowid LIMIT 1 OFFSET ?2";
- sqlite3_stmt *probe = NULL;
- if (sqlite3_prepare_v2(db, probe_sql, BM25_SQL_AUTO_LEN, &probe, NULL) == SQLITE_OK) {
- sqlite3_bind_text(probe, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN,
- MCP_SQLITE_TRANSIENT);
- sqlite3_bind_int(probe, BM25_BIND_PROJECT, BM25_INNER_LIMIT);
- int probe_step = sqlite3_step(probe);
- candidate_window_saturated = probe_step != SQLITE_DONE;
- sqlite3_finalize(probe);
- }
- }
+ /* file_pattern, kind and namespace cannot be put into the MATCH the way
+ * the label is, so they filter AFTER the candidate window. When more
+ * candidates exist beyond it, a narrow filter can find none of its rows in
+ * the top BM25_INNER_LIMIT and the search answered 0 rows although
+ * matching nodes exist. A saturated window under such a filter -- or one
+ * that held no row passing the filters at all -- is therefore ranked over
+ * every candidate instead: the FTS-first subquery plan stays, the extra
+ * work is linear in the token's match count (the saturation probe already
+ * orders every match), and the total becomes exact. The decision depends
+ * only on the query, never on the page, so offset pages stay consistent.
+ * Unfiltered and label-only queries that find rows keep the window. */
+ bool saturated = bm25_window_saturated(db, fts_query);
+ bool post_window_filter = file_like || (kind && kind[0]) || (ns && ns[0]);
+ int inner_limit = saturated && post_window_filter ? BM25_INNER_ALL : BM25_INNER_LIMIT;
+ int total = bm25_count_candidates(db, &filters, inner_limit);
+ if (saturated && total == 0 && inner_limit != BM25_INNER_ALL) {
+ inner_limit = BM25_INNER_ALL;
+ total = bm25_count_candidates(db, &filters, inner_limit);
+ }
+ bool candidate_window_saturated = saturated && inner_limit != BM25_INNER_ALL;
+ sqlite3_bind_int(stmt, BM25_BIND_INNER, inner_limit);
int row_cap = limit > 0 ? limit : BM25_DEFAULT_LIMIT;
bm25_output_row_t *rows =
diff --git a/tests/test_mcp.c b/tests/test_mcp.c
index 28d1946b7..2ef16fefe 100644
--- a/tests/test_mcp.c
+++ b/tests/test_mcp.c
@@ -4729,6 +4729,156 @@ TEST(tool_search_graph_bm25_zero_hint_names_the_label) {
PASS();
}
+/* file_pattern, kind and namespace filter after the BM25 candidate window
+ * (they cannot be put into the MATCH like the label). 2001 same-label nodes
+ * with a short path fill the window ahead of 3 that rank strictly lower (long
+ * path) and are the only ones passing each filter, so a windowed search
+ * answered 0 rows. A saturated window under such a filter is ranked over
+ * every candidate: all 3 rows, an exact total, consistent offset pages. */
+static void bm25_post_window_fixture(cbm_store_t *store, const char *project) {
+ cbm_store_upsert_project(store, project, "/tmp/bm25-post-window");
+ for (int i = 0; i < 2004; i++) {
+ bool rare = i >= 2001;
+ char qn[64];
+ snprintf(qn, sizeof(qn), "bm25-post-window.%s_%04d", rare ? "rare" : "fn", i);
+ cbm_node_t node = {
+ .project = project,
+ .label = "Function",
+ .name = "postneedle",
+ .qualified_name = qn,
+ .file_path = rare ? "deeply/nested/package/layout/registry.c" : "many.c",
+ .start_line = i + 1,
+ .end_line = i + 1,
+ .properties_json =
+ rare ? "{\"kind\":\"Deployment\",\"obj_key\":\"apps/Deployment/prod/web\"}" : NULL};
+ cbm_store_upsert_node(store, &node);
+ }
+ cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');");
+ cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "
+ "file_path) SELECT id, cbm_camel_split(name), qualified_name, "
+ "label, file_path FROM nodes;");
+}
+
+static char *bm25_post_window_search(cbm_mcp_server_t *srv, const char *filter_json, int limit,
+ int offset) {
+ char req[512];
+ snprintf(req, sizeof(req),
+ "{\"jsonrpc\":\"2.0\",\"id\":2424,\"method\":\"tools/call\","
+ "\"params\":{\"name\":\"search_graph\",\"arguments\":{"
+ "\"project\":\"bm25-post-window\",\"query\":\"postneedle\"%s,"
+ "\"limit\":%d,\"offset\":%d,\"format\":\"json\"}}}",
+ filter_json, limit, offset);
+ char *resp = cbm_mcp_server_handle(srv, req);
+ char *inner = resp ? extract_text_content(resp) : NULL;
+ free(resp);
+ return inner;
+}
+
+TEST(tool_search_graph_bm25_post_window_filters_rank_every_candidate) {
+ cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
+ ASSERT_NOT_NULL(srv);
+ cbm_store_t *store = cbm_mcp_server_store(srv);
+ ASSERT_NOT_NULL(store);
+ cbm_mcp_server_set_project(srv, "bm25-post-window");
+ bm25_post_window_fixture(store, "bm25-post-window");
+
+ static const char *const filters[] = {
+ ",\"file_pattern\":\"deeply/nested\"",
+ ",\"kind\":\"Deployment\"",
+ ",\"namespace\":\"prod\"",
+ };
+ for (size_t i = 0; i < sizeof(filters) / sizeof(filters[0]); i++) {
+ char *inner = bm25_post_window_search(srv, filters[i], 10, 0);
+ ASSERT_NOT_NULL(inner);
+ ASSERT_NOT_NULL(strstr(inner, "\"search_mode\":\"bm25\""));
+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-post-window.rare_2001\""));
+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-post-window.rare_2003\""));
+ ASSERT_NULL(strstr(inner, "\"bm25-post-window.fn_"));
+ ASSERT_NOT_NULL(strstr(inner, "\"total\":3,"));
+ ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));
+ ASSERT_NULL(strstr(inner, "candidate_window_saturated"));
+ free(inner);
+ }
+
+ /* Offset pages over the exhaustive ranking are disjoint and complete. */
+ char *page1 = bm25_post_window_search(srv, filters[0], 2, 0);
+ char *page2 = bm25_post_window_search(srv, filters[0], 2, 2);
+ ASSERT_NOT_NULL(page1);
+ ASSERT_NOT_NULL(page2);
+ ASSERT_NOT_NULL(strstr(page1, "\"returned\":2,"));
+ ASSERT_NOT_NULL(strstr(page2, "\"returned\":1,"));
+ ASSERT_NOT_NULL(strstr(page2, "\"total\":3,"));
+ for (int i = 2001; i < 2004; i++) {
+ char qn[64];
+ snprintf(qn, sizeof(qn), "\"bm25-post-window.rare_%04d\"", i);
+ ASSERT_TRUE((strstr(page1, qn) != NULL) != (strstr(page2, qn) != NULL));
+ }
+ free(page1);
+ free(page2);
+
+ /* Without a post-window filter the window stays and says so. */
+ char *inner = bm25_post_window_search(srv, "", 10, 0);
+ ASSERT_NOT_NULL(inner);
+ ASSERT_NOT_NULL(strstr(inner, "\"total\":2000,"));
+ ASSERT_NOT_NULL(strstr(inner, "\"candidate_window_saturated\":true"));
+ free(inner);
+
+ cbm_mcp_server_free(srv);
+ PASS();
+}
+
+/* An unlabelled query whose whole candidate window is File/Folder/Variable/
+ * Project rows (excluded from unlabelled ranking) found no row at all, while
+ * a matching definition sat just beyond the window: 2001 short Folder rows
+ * outrank one Function with a long path. A window with no row passing the
+ * filters is ranked over every candidate instead. */
+TEST(tool_search_graph_bm25_empty_window_ranks_every_candidate) {
+ cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
+ ASSERT_NOT_NULL(srv);
+ cbm_store_t *store = cbm_mcp_server_store(srv);
+ ASSERT_NOT_NULL(store);
+ const char *project = "bm25-empty-window";
+ cbm_mcp_server_set_project(srv, project);
+ ASSERT_EQ(cbm_store_upsert_project(store, project, "/tmp/bm25-empty-window"), CBM_STORE_OK);
+ for (int i = 0; i < 2002; i++) {
+ bool def = i == 2001;
+ char qn[64];
+ snprintf(qn, sizeof(qn), "bm25-empty-window.%s_%04d", def ? "def" : "dir", i);
+ cbm_node_t node = {.project = project,
+ .label = def ? "Function" : "Folder",
+ .name = "foldneedle",
+ .qualified_name = qn,
+ .file_path = def ? "deeply/nested/package/layout/registry.c" : "d",
+ .start_line = i + 1,
+ .end_line = i + 1};
+ ASSERT_GT(cbm_store_upsert_node(store, &node), 0);
+ }
+ ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');"),
+ CBM_STORE_OK);
+ ASSERT_EQ(cbm_store_exec(store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, "
+ "file_path) SELECT id, cbm_camel_split(name), qualified_name, "
+ "label, file_path FROM nodes;"),
+ CBM_STORE_OK);
+
+ char *resp = cbm_mcp_server_handle(
+ srv, "{\"jsonrpc\":\"2.0\",\"id\":2425,\"method\":\"tools/call\","
+ "\"params\":{\"name\":\"search_graph\",\"arguments\":{"
+ "\"project\":\"bm25-empty-window\",\"query\":\"foldneedle\",\"limit\":10,"
+ "\"format\":\"json\"}}}");
+ ASSERT_NOT_NULL(resp);
+ char *inner = extract_text_content(resp);
+ free(resp);
+ ASSERT_NOT_NULL(inner);
+ ASSERT_NOT_NULL(strstr(inner, "\"bm25-empty-window.def_2001\",\"Function\""));
+ ASSERT_NULL(strstr(inner, "\"Folder\""));
+ ASSERT_NOT_NULL(strstr(inner, "\"total\":1,"));
+ ASSERT_NOT_NULL(strstr(inner, "\"total_relation\":\"eq\""));
+ free(inner);
+
+ cbm_mcp_server_free(srv);
+ PASS();
+}
+
/* The definition whose NAME is the query ranks first: the `Table` class above
* its own methods and above test methods that repeat "table" three times; the
* `get_object_or_404` function above the test methods that contain it. */
@@ -25336,6 +25486,8 @@ SUITE(mcp) {
RUN_TEST(tool_search_graph_bm25_label_scope_keeps_rank_order);
RUN_TEST(tool_search_graph_bm25_serves_unranked_labels_when_named);
RUN_TEST(tool_search_graph_bm25_zero_hint_names_the_label);
+ RUN_TEST(tool_search_graph_bm25_post_window_filters_rank_every_candidate);
+ RUN_TEST(tool_search_graph_bm25_empty_window_ranks_every_candidate);
RUN_TEST(tool_search_graph_bm25_ranks_exact_name_first);
RUN_TEST(tool_search_graph_rejects_bm25_and_semantic_query_together);
RUN_TEST(tool_search_graph_semantic_ceiling_never_emits_unusable_continuation);