Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
perf(py-lsp): O(1) type membership in the shared Python cross-registry build
Nelson Spencee36930742a63
diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.cindex 8a0396115..961e35f15 100644--- a/internal/cbm/lsp/py_lsp.c+++ b/internal/cbm/lsp/py_lsp.c@@ -13,6 +13,7 @@#include "py_lsp.h"#include "../cbm.h"#include "../helpers.h"+#include "../../../src/foundation/hash_table.h"#include "tree_sitter/api.h"#include <ctype.h>#include <stdint.h>@@ -4973,10 +4974,36 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) {return out;}+/* Qualified names of reg->types[0..noted), for a caller that registers into a+ * registry it does not finalize until the end (cbm_py_build_cross_registry):+ * that registry's own lookup scans every type. qns == NULL probes the+ * registry instead. */+typedef struct {+ CBMHashTable *qns;+ int noted;+} PyKnownTypes;++/* Note the types added since the last call. Run after every add_type, so an+ * add that failed silently cannot desync set and registry. If the set cannot+ * grow it is dropped and probes fall back to the registry. */+static void py_note_known_types(PyKnownTypes *known, const CBMTypeRegistry *reg) {+ for (; known && known->qns && known->noted < reg->type_count; known->noted++) {+ const char *qn = reg->types[known->noted].qualified_name;+ if (!qn || cbm_ht_has(known->qns, qn))+ continue;+ cbm_ht_set(known->qns, qn, (void *)(uintptr_t)(known->noted + 1));+ if (!cbm_ht_has(known->qns, qn)) {+ cbm_ht_free(known->qns);+ known->qns = NULL;+ }+ }+}+/* Build a registry from CBMLSPDef[] supplied by the caller — covers both- * the source file's own defs and cross-file referenced defs. */+ * the source file's own defs and cross-file referenced defs. known is NULL+ * except for the shared Tier-2 build. */static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg,- CBMLSPDef *defs, int def_count) {+ CBMLSPDef *defs, int def_count, PyKnownTypes *known) {/* Pass 1: types only — the method pass probes the registry per Method def* (receiver auto-registration), which is a LINEAR scan pre-finalize:* O(methods x types) per file (same quadratic as php_register_lsp_defs;@@ -4999,6 +5026,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRert.method_names = py_split_pipe(arena, d->method_names_str);}cbm_registry_add_type(reg, rt);+ py_note_known_types(known, reg);}}/* idx_arena == NULL skips the mid-build finalize: the tier-2 builder@@ -5046,13 +5074,17 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeReif (strcmp(d->label, "Method") == 0 && d->receiver_type && d->receiver_type[0]) {rf.receiver_type = d->receiver_type; /* borrowed */- if (!cbm_registry_lookup_type(reg, rf.receiver_type)) {+ bool recv_known = known && known->qns+ ? cbm_ht_has(known->qns, rf.receiver_type)+ : cbm_registry_lookup_type(reg, rf.receiver_type) != NULL;+ if (!recv_known) {CBMRegisteredType auto_t;memset(&auto_t, 0, sizeof(auto_t));auto_t.qualified_name = rf.receiver_type;const char *dot = strrchr(d->receiver_type, '.');auto_t.short_name = dot ? dot + 1 : rf.receiver_type; /* borrowed substring */cbm_registry_add_type(reg, auto_t);+ py_note_known_types(known, reg);}}cbm_registry_add_func(reg, rf);@@ -5092,7 +5124,7 @@ void cbm_run_py_lsp_cross(CBMArena *arena, const char *source, int source_len,/* Index allocations go to a per-call scratch arena (see php_lsp_cross). */CBMArena idx_arena;cbm_arena_init(&idx_arena);- py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count);+ py_register_lsp_defs(arena, &idx_arena, ®, defs, def_count, NULL);py_mark_ambiguous_callable_bindings(®);/* Finalize registry — O(1) lookups. See go_lsp.c "3c. Finalize"@@ -5126,14 +5158,24 @@ CBMTypeRegistry *cbm_py_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, icbm_registry_init(reg, arena);cbm_python_stdlib_register(reg, arena);+ /* The registry is not finalized until the end, so probing it for each+ * Method's receiver scanned every type: O(methods x types), single-threaded+ * (~54 s of lsp_cross_prepare on a large Python tree). Probe a set of+ * the registered type QNs instead; registration order is unchanged. Sized+ * for the stdlib seed, not def_count (the mixed all-language count). NULL+ * on allocation failure: the registry probe runs as before. */+ PyKnownTypes known = {cbm_ht_create((uint32_t)reg->type_count), 0};+ py_note_known_types(&known, reg);+/* Filter to Python defs only — defs[] is mixed-language all_defs. */for (int i = 0; i < def_count; i++) {CBMLSPDef *d = &defs[i];if (d->lang != CBM_LANG_PYTHON)continue;/* Reuse the existing register fn on a single-def slice (n=1 inline). */- py_register_lsp_defs(arena, NULL, reg, d, 1);+ py_register_lsp_defs(arena, NULL, reg, d, 1, &known);}+ cbm_ht_free(known.qns);py_mark_ambiguous_callable_bindings(reg);cbm_registry_finalize(reg);diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.cindex 3aaadee51..798c80a7b 100644--- a/tests/test_py_lsp.c+++ b/tests/test_py_lsp.c@@ -637,6 +637,91 @@ TEST(pylsp_fused_self_attr_chain_via_overlay) {PASS();}+/* Characterization of the shared Tier-2 build order. Defs register one at a+ * time in defs[] order, so a Method seen before its Class auto-registers a+ * placeholder receiver type that stays ahead of the Class entry, and a Method+ * whose receiver is already known adds no type. Any change to how the build+ * probes for known receivers must keep this exact sequence. */+TEST(pylsp_cross_registry_interleaved_order) {+ CBMArena arena;+ cbm_arena_init(&arena);+ CBMTypeRegistry *base = cbm_py_build_cross_registry(&arena, NULL, 0);+ ASSERT_NOT_NULL(base);+ int tb = base->type_count, fb = base->func_count;++ CBMLSPDef defs[] = {+ {.qualified_name = "t.m.A.m",+ .short_name = "m",+ .label = "Method",+ .receiver_type = "t.m.A",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ {.qualified_name = "t.m.A",+ .short_name = "A",+ .label = "Class",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ {.qualified_name = "t.m.B",+ .short_name = "B",+ .label = "Class",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ {.qualified_name = "t.m.B.f",+ .short_name = "f",+ .label = "Method",+ .receiver_type = "t.m.B",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ {.qualified_name = "t.m.C.g",+ .short_name = "g",+ .label = "Method",+ .receiver_type = "t.m.C",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ /* Receiver auto-registered by C.g just above: no second placeholder. */+ {.qualified_name = "t.m.C.h",+ .short_name = "h",+ .label = "Method",+ .receiver_type = "t.m.C",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_PYTHON},+ /* Non-Python defs are skipped by the builder. */+ {.qualified_name = "t.m.D",+ .short_name = "D",+ .label = "Class",+ .def_module_qn = "t.m",+ .lang = CBM_LANG_GO},+ };+ CBMTypeRegistry *reg =+ cbm_py_build_cross_registry(&arena, defs, (int)(sizeof(defs) / sizeof(defs[0])));+ ASSERT_NOT_NULL(reg);++ /* Types: placeholder A (from A.m), Class A, Class B, placeholder C. */+ ASSERT_EQ(reg->type_count, tb + 4);+ ASSERT_TRUE(reg->types[tb].qualified_name == defs[0].receiver_type);+ ASSERT_STR_EQ(reg->types[tb].short_name, "A");+ ASSERT_TRUE(reg->types[tb + 1].qualified_name == defs[1].qualified_name);+ ASSERT_TRUE(reg->types[tb + 2].qualified_name == defs[2].qualified_name);+ ASSERT_TRUE(reg->types[tb + 3].qualified_name == defs[4].receiver_type);+ ASSERT_STR_EQ(reg->types[tb + 3].short_name, "C");++ /* Funcs keep defs[] order. */+ ASSERT_EQ(reg->func_count, fb + 4);+ ASSERT_STR_EQ(reg->funcs[fb].qualified_name, "t.m.A.m");+ ASSERT_STR_EQ(reg->funcs[fb + 1].qualified_name, "t.m.B.f");+ ASSERT_STR_EQ(reg->funcs[fb + 2].qualified_name, "t.m.C.g");+ ASSERT_STR_EQ(reg->funcs[fb + 2].receiver_type, "t.m.C");+ ASSERT_STR_EQ(reg->funcs[fb + 3].qualified_name, "t.m.C.h");++ const CBMRegisteredType *a = cbm_registry_lookup_type(reg, "t.m.A");+ ASSERT_NOT_NULL(a);+ ASSERT_STR_EQ(a->qualified_name, "t.m.A");+ ASSERT_NULL(cbm_registry_lookup_type(reg, "t.m.D"));++ cbm_arena_destroy(&arena);+ PASS();+}+/* Issue #228: a class/static method invoked directly on a CROSS-FILE imported* class name — ActionRecordX.build_from_text(...) — produced no CALLS edge, so* the method showed in/out degree 0 and was flagged as dead code. Distinct from@@ -2194,6 +2279,7 @@ SUITE(py_lsp) {/* Phase 9 — cross-file + batch */RUN_TEST(pylsp_crossfile_method_dispatch);RUN_TEST(pylsp_fused_self_attr_chain_via_overlay);+ RUN_TEST(pylsp_cross_registry_interleaved_order);RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228);RUN_TEST(pylsp_crossfile_inheritance);RUN_TEST(pylsp_batch_two_files);diff --git a/tests/test_py_lsp_scale.c b/tests/test_py_lsp_scale.cindex f20e76275..eee1575fd 100644--- a/tests/test_py_lsp_scale.c+++ b/tests/test_py_lsp_scale.c@@ -111,6 +111,72 @@ TEST(pylsp_scale_linear_growth) {PASS();}+/* Time cbm_py_build_cross_registry over n classes, each followed by two+ * methods (extraction order): every method probes the registry for its+ * receiver while the registry is still being built. */+static double measure_cross_registry(int n_classes, int *out_types) {+ CBMArena arena;+ cbm_arena_init(&arena);+ int n_defs = n_classes * 3;+ CBMLSPDef *defs = (CBMLSPDef *)cbm_arena_calloc(&arena, (size_t)n_defs * sizeof(*defs));+ if (!defs) {+ cbm_arena_destroy(&arena);+ return -1.0;+ }+ for (int i = 0; i < n_classes; i++) {+ CBMLSPDef *cls = &defs[i * 3];+ cls->qualified_name = cbm_arena_sprintf(&arena, "scale.mod.Cls%d", i);+ cls->short_name = cls->qualified_name + strlen("scale.mod.");+ cls->label = "Class";+ cls->def_module_qn = "scale.mod";+ cls->lang = CBM_LANG_PYTHON;+ for (int j = 1; j <= 2; j++) {+ CBMLSPDef *m = &defs[i * 3 + j];+ m->qualified_name = cbm_arena_sprintf(&arena, "%s.m%d", cls->qualified_name, j);+ m->short_name = j == 1 ? "m1" : "m2";+ m->label = "Method";+ m->receiver_type = cls->qualified_name;+ m->def_module_qn = "scale.mod";+ m->lang = CBM_LANG_PYTHON;+ }+ }+ struct timespec t0, t1;+ clock_gettime(CLOCK_MONOTONIC, &t0);+ CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, n_defs);+ clock_gettime(CLOCK_MONOTONIC, &t1);+ if (out_types)+ *out_types = reg ? reg->type_count : -1;+ cbm_arena_destroy(&arena);+ return elapsed_ms(t0, t1);+}++/* The shared build registers defs one at a time into an unfinalized registry;+ * a receiver probe that scans every type registered so far makes the build+ * O(methods x types). 16x input: linear ~16x time, quadratic well over 100x. */+TEST(pylsp_scale_cross_registry_build_linear) {+ int ty_small = 0, ty_large = 0;+ /* Best of three for the small size: it is short enough for host noise. */+ double t_small = measure_cross_registry(1000, &ty_small);+ for (int k = 0; k < 2; k++) {+ double t = measure_cross_registry(1000, NULL);+ if (t >= 0.0 && t < t_small)+ t_small = t;+ }+ double t_large = measure_cross_registry(16000, &ty_large);+ printf(" cross registry: 1000 classes=%.1fms (types=%d) 16000 classes=%.1fms "+ "(types=%d)\n",+ t_small, ty_small, t_large, ty_large);+ ASSERT(t_small >= 0.0 && t_large >= 0.0);+ ASSERT_EQ(ty_large - ty_small, 15000);+ if (t_small > 0.5) {+ double ratio = t_large / t_small;+ printf(" cross registry ratio 16000/1000: %.1fx (linear ~16x)\n", ratio);+ ASSERT(ratio < 48.0);+ }+ PASS();+}+SUITE(py_lsp_scale) {RUN_TEST(pylsp_scale_linear_growth);+ RUN_TEST(pylsp_scale_cross_registry_build_linear);}