Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(semantic): read JSON strings with their escapes; recalibrate p
Martin Vogel3d559b3554ac
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.cindex 17e4f55ad..da131d3a9 100644--- a/src/pipeline/pass_semantic_edges.c+++ b/src/pipeline/pass_semantic_edges.c@@ -327,29 +327,9 @@ static const char *file_ext(const char *path) {return dot ? dot : "";}-/* Extract a JSON string value by key (simple strstr-based, no full parse). */+/* Extract a JSON string value by key, escapes honoured (cbm_sem_json_str). */static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize) {- if (!json || !key) {- return NULL;- }- char search[CBM_SZ_64];- snprintf(search, sizeof(search), "\"%s\":\"", key);- const char *start = strstr(json, search);- if (!start) {- return NULL;- }- start += strlen(search);- const char *end = strchr(start, '"');- if (!end) {- return NULL;- }- int len = (int)(end - start);- if (len >= bufsize) {- len = bufsize - SKIP_ONE;- }- memcpy(buf, start, (size_t)len);- buf[len] = '\0';- return buf;+ return cbm_sem_json_str(json, key, buf, bufsize);}/* Extract a JSON array of strings by key. Returns count. */diff --git a/src/semantic/semantic.c b/src/semantic/semantic.cindex d894d941f..907b03a30 100644--- a/src/semantic/semantic.c+++ b/src/semantic/semantic.c@@ -1798,14 +1798,60 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,return true;}-/* Score bands at the quartiles of the admitted scores, each band's p the- * judged share of related pairs (2026-10-07: 120 blind-judged admitted pairs- * from held-out repositories in six languages, three reading rounds; the- * curve is monotone as measured). Ships as the edge property "p". The cuts- * (measured 0.777752, 0.805782, 0.851543) are at the stored score's three- * decimals, so one shown score always carries one p. */-static const float SEM_P_CUTS[] = {0.778F, 0.806F, 0.852F};-static const float SEM_P_BANDS[] = {0.556F, 0.640F, 0.731F, 0.857F};+/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw:+ * a docstring holding a quote lost everything after it, and "\nreturn" gave+ * the token "nreturn". */+const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) {+ if (!json || !key || !buf || bufsize <= 0) {+ return NULL;+ }+ char search[CBM_SZ_64];+ snprintf(search, sizeof(search), "\"%s\":\"", key);+ const char *p = strstr(json, search);+ if (!p) {+ return NULL;+ }+ p += strlen(search);+ int n = 0;+ for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) {+ char c = *p;+ if (c == '\\' && p[SKIP_ONE]) {+ p++;+ switch (*p) {+ case 'n':+ case 't':+ case 'r':+ case 'b':+ case 'f':+ c = ' ';+ break;+ case 'u':+ c = ' ';+ for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) {+ p++;+ }+ break;+ default:+ c = *p; /* \" \\ \/ */+ break;+ }+ }+ buf[n++] = c;+ }+ buf[n] = '\0';+ return buf;+}++/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at+ * the stored score's three decimals, so one shown score always carries one p),+ * each band's p the judged share of related pairs, pooled where a higher band+ * judged lower: the upper three pool to one. Judged 2026-10-07 in eleven+ * held-out repositories: the pairs this scoring admits that the previous one+ * did not (100, blind, three reading rounds) and the earlier samples' pairs it+ * keeps (65), each weighted by its share of the admitted set. Ships as the+ * edge property "p". */+static const float SEM_P_CUTS[] = {0.773F};+static const float SEM_P_BANDS[] = {0.626F, 0.760F};float cbm_sem_calibrated_p(float score) {size_t band = 0;diff --git a/src/semantic/semantic.h b/src/semantic/semantic.hindex 34f49c0c5..143909ef6 100644--- a/src/semantic/semantic.h+++ b/src/semantic/semantic.h@@ -253,6 +253,12 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,const bool *eligible, int n, int max_edges, int *counts,bool *admitted);+/* The string value of "key" in a properties JSON object, unescaped into buf+ * (cut at bufsize - 1); NULL when the key is absent. Escapes are honoured:+ * \" \\ \/ keep their character, \n \t \r \b \f and \uXXXX become a space (a+ * separator for the tokenizer), so a value holding a quote is read whole. */+const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize);+/* The probability that an admitted SEMANTICALLY_RELATED pair is related, by* its score as stored (three decimals): the judged precision of blind-judged* held-out samples (the edge property "p"). */diff --git a/tests/test_semantic.c b/tests/test_semantic.cindex 8ae161ed6..98dd0b0ec 100644--- a/tests/test_semantic.c+++ b/tests/test_semantic.c@@ -496,16 +496,14 @@ TEST(sem_admit_best_first_keeps_the_best) {/* p by the stored (three-decimal) score: the judged bands, monotone. */TEST(sem_calibrated_p_bands) {- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.556F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.777F), 0.556F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.778F), 0.640F, 0.0);- /* the cut sits at the stored precision (0.778), not the raw quartile- * 0.777752: a stored "0.778" and a stored "0.777" never share a band */- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7779F), 0.556F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.805F), 0.640F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.806F), 0.731F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.852F), 0.857F, 0.0);- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.857F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.626F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.772F), 0.626F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.773F), 0.760F, 0.0);+ /* the cut sits at the stored precision (0.773), not the raw quartile:+ * a stored "0.773" and a stored "0.772" never share a band */+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7729F), 0.626F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.838F), 0.760F, 0.0);+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.760F, 0.0);float last = 0.0F;for (int milli = 750; milli <= 1000; milli++) {float p = cbm_sem_calibrated_p((float)milli / 1000.0F);@@ -515,6 +513,24 @@ TEST(sem_calibrated_p_bands) {PASS();}+/* A properties string holding a quote is read whole, and escapes are not+ * glued to the next word (until 2026-10 the read stopped at the first quote+ * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */+TEST(sem_json_str_honours_escapes) {+ static const char json[] =+ "{\"docstring\":\"Say \\\"hi\\\" to\\nthe user \\u00e9 now\",\"bt\":\"x\"}";+ char buf[64];+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", buf, sizeof(buf)));+ ASSERT_STR_EQ(buf, "Say \"hi\" to the user now");+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "bt", buf, sizeof(buf)));+ ASSERT_STR_EQ(buf, "x");+ ASSERT_NULL(cbm_sem_json_str(json, "absent", buf, sizeof(buf)));+ char small[6];+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", small, sizeof(small)));+ ASSERT_STR_EQ(small, "Say \""); /* cut at the buffer, still terminated */+ PASS();+}+TEST(sem_corpus_add_null_doc) {cbm_sem_corpus_t *c = cbm_sem_corpus_new();ASSERT_NOT_NULL(c);@@ -695,6 +711,7 @@ SUITE(semantic) {RUN_TEST(sem_combine_weighs_signals);RUN_TEST(sem_admit_best_first_keeps_the_best);RUN_TEST(sem_calibrated_p_bands);+ RUN_TEST(sem_json_str_honours_escapes);RUN_TEST(sem_corpus_add_null_doc);RUN_TEST(sem_corpus_free_null);RUN_TEST(sem_get_config_defaults);