Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

perf(py-lsp): O(1) type membership in the shared Python cross-registry build

Nelson Spencee36930742a63
diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c
index 8a0396115..961e35f15 100644
--- a/internal/cbm/lsp/py_lsp.c
+++ b/internal/cbm/lsp/py_lsp.c
@@ -13,6 +13,7 @@
#include "py_lsp.h"
#include "../cbm.h"
#include "../helpers.h"
+#include "../../../src/foundation/hash_table.h"
#include "tree_sitter/api.h"
#include <ctype.h>
#include <stdint.h>
@@ -4973,10 +4974,36 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) {
return out;
}
+/* Qualified names of reg->types[0..noted), for a caller that registers into a
+ * registry it does not finalize until the end (cbm_py_build_cross_registry):
+ * that registry's own lookup scans every type. qns == NULL probes the
+ * registry instead. */
+typedef struct {
+ CBMHashTable *qns;
+ int noted;
+} PyKnownTypes;
+
+/* Note the types added since the last call. Run after every add_type, so an
+ * add that failed silently cannot desync set and registry. If the set cannot
+ * grow it is dropped and probes fall back to the registry. */
+static void py_note_known_types(PyKnownTypes *known, const CBMTypeRegistry *reg) {
+ for (; known && known->qns && known->noted < reg->type_count; known->noted++) {
+ const char *qn = reg->types[known->noted].qualified_name;
+ if (!qn || cbm_ht_has(known->qns, qn))
+ continue;
+ cbm_ht_set(known->qns, qn, (void *)(uintptr_t)(known->noted + 1));
+ if (!cbm_ht_has(known->qns, qn)) {
+ cbm_ht_free(known->qns);
+ known->qns = NULL;
+ }
+ }
+}
+
/* Build a registry from CBMLSPDef[] supplied by the caller — covers both
- * the source file's own defs and cross-file referenced defs. */
+ * the source file's own defs and cross-file referenced defs. known is NULL
+ * except for the shared Tier-2 build. */
static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg,
- CBMLSPDef *defs, int def_count) {
+ CBMLSPDef *defs, int def_count, PyKnownTypes *known) {
/* Pass 1: types only — the method pass probes the registry per Method def
* (receiver auto-registration), which is a LINEAR scan pre-finalize:
* O(methods x types) per file (same quadratic as php_register_lsp_defs;
@@ -4999,6 +5026,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe
rt.method_names = py_split_pipe(arena, d->method_names_str);
}
cbm_registry_add_type(reg, rt);
+ py_note_known_types(known, reg);
}
}
/* idx_arena == NULL skips the mid-build finalize: the tier-2 builder
@@ -5046,13 +5074,17 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe
if (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) {
rf.receiver_type = d->receiver_type; /* borrowed */
- if (!cbm_registry_lookup_type(reg, rf.receiver_type)) {
+ bool recv_known = known && known->qns
+ ? cbm_ht_has(known->qns, rf.receiver_type)
+ : cbm_registry_lookup_type(reg, rf.receiver_type) != NULL;
+ if (!recv_known) {
CBMRegisteredType auto_t;
memset(&auto_t, 0, sizeof(auto_t));
auto_t.qualified_name = rf.receiver_type;
const char *dot = strrchr(d->receiver_type, '.');
auto_t.short_name = dot ? dot + 1 : rf.receiver_type; /* borrowed substring */
cbm_registry_add_type(reg, auto_t);
+ py_note_known_types(known, reg);
}
}
cbm_registry_add_func(reg, rf);
@@ -5092,7 +5124,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len,
/* Index allocations go to a per-call scratch arena (see php_lsp_cross). */
CBMArena idx_arena;
cbm_arena_init(&idx_arena);
- py_register_lsp_defs(arena, &idx_arena, &reg, defs, def_count);
+ py_register_lsp_defs(arena, &idx_arena, &reg, defs, def_count, NULL);
py_mark_ambiguous_callable_bindings(&reg);
/* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"
@@ -5126,14 +5158,24 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i
cbm_registry_init(reg, arena);
cbm_python_stdlib_register(reg, arena);
+ /* The registry is not finalized until the end, so probing it for each
+ * Method's receiver scanned every type: O(methods x types), single-threaded
+ * (~54 s of lsp_cross_prepare on a large Python tree). Probe a set of
+ * the registered type QNs instead; registration order is unchanged. Sized
+ * for the stdlib seed, not def_count (the mixed all-language count). NULL
+ * on allocation failure: the registry probe runs as before. */
+ PyKnownTypes known = {cbm_ht_create((uint32_t)reg->type_count), 0};
+ py_note_known_types(&known, reg);
+
/* Filter to Python defs only — defs[] is mixed-language all_defs. */
for (int i = 0; i < def_count; i++) {
CBMLSPDef *d = &defs[i];
if (d->lang != CBM_LANG_PYTHON)
continue;
/* Reuse the existing register fn on a single-def slice (n=1 inline). */
- py_register_lsp_defs(arena, NULL, reg, d, 1);
+ py_register_lsp_defs(arena, NULL, reg, d, 1, &known);
}
+ cbm_ht_free(known.qns);
py_mark_ambiguous_callable_bindings(reg);
cbm_registry_finalize(reg);
diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c
index 3aaadee51..798c80a7b 100644
--- a/tests/test_py_lsp.c
+++ b/tests/test_py_lsp.c
@@ -637,6 +637,91 @@ TEST(pylsp_fused_self_attr_chain_via_overlay) {
PASS();
}
+/* Characterization of the shared Tier-2 build order. Defs register one at a
+ * time in defs[] order, so a Method seen before its Class auto-registers a
+ * placeholder receiver type that stays ahead of the Class entry, and a Method
+ * whose receiver is already known adds no type. Any change to how the build
+ * probes for known receivers must keep this exact sequence. */
+TEST(pylsp_cross_registry_interleaved_order) {
+ CBMArena arena;
+ cbm_arena_init(&arena);
+ CBMTypeRegistry *base = cbm_py_build_cross_registry(&arena, NULL, 0);
+ ASSERT_NOT_NULL(base);
+ int tb = base->type_count, fb = base->func_count;
+
+ CBMLSPDef defs[] = {
+ {.qualified_name = "t.m.A.m",
+ .short_name = "m",
+ .label = "Method",
+ .receiver_type = "t.m.A",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ {.qualified_name = "t.m.A",
+ .short_name = "A",
+ .label = "Class",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ {.qualified_name = "t.m.B",
+ .short_name = "B",
+ .label = "Class",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ {.qualified_name = "t.m.B.f",
+ .short_name = "f",
+ .label = "Method",
+ .receiver_type = "t.m.B",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ {.qualified_name = "t.m.C.g",
+ .short_name = "g",
+ .label = "Method",
+ .receiver_type = "t.m.C",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ /* Receiver auto-registered by C.g just above: no second placeholder. */
+ {.qualified_name = "t.m.C.h",
+ .short_name = "h",
+ .label = "Method",
+ .receiver_type = "t.m.C",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_PYTHON},
+ /* Non-Python defs are skipped by the builder. */
+ {.qualified_name = "t.m.D",
+ .short_name = "D",
+ .label = "Class",
+ .def_module_qn = "t.m",
+ .lang = CBM_LANG_GO},
+ };
+ CBMTypeRegistry *reg =
+ cbm_py_build_cross_registry(&arena, defs, (int)(sizeof(defs) / sizeof(defs[0])));
+ ASSERT_NOT_NULL(reg);
+
+ /* Types: placeholder A (from A.m), Class A, Class B, placeholder C. */
+ ASSERT_EQ(reg->type_count, tb + 4);
+ ASSERT_TRUE(reg->types[tb].qualified_name == defs[0].receiver_type);
+ ASSERT_STR_EQ(reg->types[tb].short_name, "A");
+ ASSERT_TRUE(reg->types[tb + 1].qualified_name == defs[1].qualified_name);
+ ASSERT_TRUE(reg->types[tb + 2].qualified_name == defs[2].qualified_name);
+ ASSERT_TRUE(reg->types[tb + 3].qualified_name == defs[4].receiver_type);
+ ASSERT_STR_EQ(reg->types[tb + 3].short_name, "C");
+
+ /* Funcs keep defs[] order. */
+ ASSERT_EQ(reg->func_count, fb + 4);
+ ASSERT_STR_EQ(reg->funcs[fb].qualified_name, "t.m.A.m");
+ ASSERT_STR_EQ(reg->funcs[fb + 1].qualified_name, "t.m.B.f");
+ ASSERT_STR_EQ(reg->funcs[fb + 2].qualified_name, "t.m.C.g");
+ ASSERT_STR_EQ(reg->funcs[fb + 2].receiver_type, "t.m.C");
+ ASSERT_STR_EQ(reg->funcs[fb + 3].qualified_name, "t.m.C.h");
+
+ const CBMRegisteredType *a = cbm_registry_lookup_type(reg, "t.m.A");
+ ASSERT_NOT_NULL(a);
+ ASSERT_STR_EQ(a->qualified_name, "t.m.A");
+ ASSERT_NULL(cbm_registry_lookup_type(reg, "t.m.D"));
+
+ cbm_arena_destroy(&arena);
+ PASS();
+}
+
/* Issue #228: a class/static method invoked directly on a CROSS-FILE imported
* class name — ActionRecordX.build_from_text(...) — produced no CALLS edge, so
* the method showed in/out degree 0 and was flagged as dead code. Distinct from
@@ -2194,6 +2279,7 @@ SUITE(py_lsp) {
/* Phase 9 — cross-file + batch */
RUN_TEST(pylsp_crossfile_method_dispatch);
RUN_TEST(pylsp_fused_self_attr_chain_via_overlay);
+ RUN_TEST(pylsp_cross_registry_interleaved_order);
RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228);
RUN_TEST(pylsp_crossfile_inheritance);
RUN_TEST(pylsp_batch_two_files);
diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.c
index f20e76275..eee1575fd 100644
--- a/tests/test_py_lsp_scale.c
+++ b/tests/test_py_lsp_scale.c
@@ -111,6 +111,72 @@ TEST(pylsp_scale_linear_growth) {
PASS();
}
+/* Time cbm_py_build_cross_registry over n classes, each followed by two
+ * methods (extraction order): every method probes the registry for its
+ * receiver while the registry is still being built. */
+static double measure_cross_registry(int n_classes, int *out_types) {
+ CBMArena arena;
+ cbm_arena_init(&arena);
+ int n_defs = n_classes * 3;
+ CBMLSPDef *defs = (CBMLSPDef *)cbm_arena_calloc(&arena, (size_t)n_defs * sizeof(*defs));
+ if (!defs) {
+ cbm_arena_destroy(&arena);
+ return -1.0;
+ }
+ for (int i = 0; i < n_classes; i++) {
+ CBMLSPDef *cls = &defs[i * 3];
+ cls->qualified_name = cbm_arena_sprintf(&arena, "scale.mod.Cls%d", i);
+ cls->short_name = cls->qualified_name + strlen("scale.mod.");
+ cls->label = "Class";
+ cls->def_module_qn = "scale.mod";
+ cls->lang = CBM_LANG_PYTHON;
+ for (int j = 1; j <= 2; j++) {
+ CBMLSPDef *m = &defs[i * 3 + j];
+ m->qualified_name = cbm_arena_sprintf(&arena, "%s.m%d", cls->qualified_name, j);
+ m->short_name = j == 1 ? "m1" : "m2";
+ m->label = "Method";
+ m->receiver_type = cls->qualified_name;
+ m->def_module_qn = "scale.mod";
+ m->lang = CBM_LANG_PYTHON;
+ }
+ }
+ struct timespec t0, t1;
+ clock_gettime(CLOCK_MONOTONIC, &t0);
+ CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, n_defs);
+ clock_gettime(CLOCK_MONOTONIC, &t1);
+ if (out_types)
+ *out_types = reg ? reg->type_count : -1;
+ cbm_arena_destroy(&arena);
+ return elapsed_ms(t0, t1);
+}
+
+/* The shared build registers defs one at a time into an unfinalized registry;
+ * a receiver probe that scans every type registered so far makes the build
+ * O(methods x types). 16x input: linear ~16x time, quadratic well over 100x. */
+TEST(pylsp_scale_cross_registry_build_linear) {
+ int ty_small = 0, ty_large = 0;
+ /* Best of three for the small size: it is short enough for host noise. */
+ double t_small = measure_cross_registry(1000, &ty_small);
+ for (int k = 0; k < 2; k++) {
+ double t = measure_cross_registry(1000, NULL);
+ if (t >= 0.0 && t < t_small)
+ t_small = t;
+ }
+ double t_large = measure_cross_registry(16000, &ty_large);
+ printf(" cross registry: 1000 classes=%.1fms (types=%d) 16000 classes=%.1fms "
+ "(types=%d)\n",
+ t_small, ty_small, t_large, ty_large);
+ ASSERT(t_small >= 0.0 && t_large >= 0.0);
+ ASSERT_EQ(ty_large - ty_small, 15000);
+ if (t_small > 0.5) {
+ double ratio = t_large / t_small;
+ printf(" cross registry ratio 16000/1000: %.1fx (linear ~16x)\n", ratio);
+ ASSERT(ratio < 48.0);
+ }
+ PASS();
+}
+
SUITE(py_lsp_scale) {
RUN_TEST(pylsp_scale_linear_growth);
+ RUN_TEST(pylsp_scale_cross_registry_build_linear);
}