Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(semantic): read JSON strings with their escapes; recalibrate p

Martin Vogel3d559b3554ac
diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c
index 17e4f55ad..da131d3a9 100644
--- a/src/pipeline/pass_semantic_edges.c
+++ b/src/pipeline/pass_semantic_edges.c
@@ -327,29 +327,9 @@ static const char *file_ext(const char *path) {
return dot ? dot : "";
}
-/* Extract a JSON string value by key (simple strstr-based, no full parse). */
+/* Extract a JSON string value by key, escapes honoured (cbm_sem_json_str). */
static const char *json_str_value(const char *json, const char *key, char *buf, int bufsize) {
- if (!json || !key) {
- return NULL;
- }
- char search[CBM_SZ_64];
- snprintf(search, sizeof(search), "\"%s\":\"", key);
- const char *start = strstr(json, search);
- if (!start) {
- return NULL;
- }
- start += strlen(search);
- const char *end = strchr(start, '"');
- if (!end) {
- return NULL;
- }
- int len = (int)(end - start);
- if (len >= bufsize) {
- len = bufsize - SKIP_ONE;
- }
- memcpy(buf, start, (size_t)len);
- buf[len] = '\0';
- return buf;
+ return cbm_sem_json_str(json, key, buf, bufsize);
}
/* Extract a JSON array of strings by key. Returns count. */
diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c
index d894d941f..907b03a30 100644
--- a/src/semantic/semantic.c
+++ b/src/semantic/semantic.c
@@ -1798,14 +1798,60 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
return true;
}
-/* Score bands at the quartiles of the admitted scores, each band's p the
- * judged share of related pairs (2026-10-07: 120 blind-judged admitted pairs
- * from held-out repositories in six languages, three reading rounds; the
- * curve is monotone as measured). Ships as the edge property "p". The cuts
- * (measured 0.777752, 0.805782, 0.851543) are at the stored score's three
- * decimals, so one shown score always carries one p. */
-static const float SEM_P_CUTS[] = {0.778F, 0.806F, 0.852F};
-static const float SEM_P_BANDS[] = {0.556F, 0.640F, 0.731F, 0.857F};
+/* Until 2026-10 the reader stopped at the first '"' and kept escapes raw:
+ * a docstring holding a quote lost everything after it, and "\nreturn" gave
+ * the token "nreturn". */
+const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize) {
+ if (!json || !key || !buf || bufsize <= 0) {
+ return NULL;
+ }
+ char search[CBM_SZ_64];
+ snprintf(search, sizeof(search), "\"%s\":\"", key);
+ const char *p = strstr(json, search);
+ if (!p) {
+ return NULL;
+ }
+ p += strlen(search);
+ int n = 0;
+ for (; *p && *p != '"' && n + SKIP_ONE < bufsize; p++) {
+ char c = *p;
+ if (c == '\\' && p[SKIP_ONE]) {
+ p++;
+ switch (*p) {
+ case 'n':
+ case 't':
+ case 'r':
+ case 'b':
+ case 'f':
+ c = ' ';
+ break;
+ case 'u':
+ c = ' ';
+ for (int k = 0; k < 4 && isxdigit((unsigned char)p[SKIP_ONE]); k++) {
+ p++;
+ }
+ break;
+ default:
+ c = *p; /* \" \\ \/ */
+ break;
+ }
+ }
+ buf[n++] = c;
+ }
+ buf[n] = '\0';
+ return buf;
+}
+
+/* Score bands at the quartiles of the admitted scores (0.773, 0.802, 0.838 at
+ * the stored score's three decimals, so one shown score always carries one p),
+ * each band's p the judged share of related pairs, pooled where a higher band
+ * judged lower: the upper three pool to one. Judged 2026-10-07 in eleven
+ * held-out repositories: the pairs this scoring admits that the previous one
+ * did not (100, blind, three reading rounds) and the earlier samples' pairs it
+ * keeps (65), each weighted by its share of the admitted set. Ships as the
+ * edge property "p". */
+static const float SEM_P_CUTS[] = {0.773F};
+static const float SEM_P_BANDS[] = {0.626F, 0.760F};
float cbm_sem_calibrated_p(float score) {
size_t band = 0;
diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h
index 34f49c0c5..143909ef6 100644
--- a/src/semantic/semantic.h
+++ b/src/semantic/semantic.h
@@ -253,6 +253,12 @@ bool cbm_sem_admit_best_first(const float *scores, const int *fa, const int *fb,
const bool *eligible, int n, int max_edges, int *counts,
bool *admitted);
+/* The string value of "key" in a properties JSON object, unescaped into buf
+ * (cut at bufsize - 1); NULL when the key is absent. Escapes are honoured:
+ * \" \\ \/ keep their character, \n \t \r \b \f and \uXXXX become a space (a
+ * separator for the tokenizer), so a value holding a quote is read whole. */
+const char *cbm_sem_json_str(const char *json, const char *key, char *buf, int bufsize);
+
/* The probability that an admitted SEMANTICALLY_RELATED pair is related, by
* its score as stored (three decimals): the judged precision of blind-judged
* held-out samples (the edge property "p"). */
diff --git a/tests/test_semantic.c b/tests/test_semantic.c
index 8ae161ed6..98dd0b0ec 100644
--- a/tests/test_semantic.c
+++ b/tests/test_semantic.c
@@ -496,16 +496,14 @@ TEST(sem_admit_best_first_keeps_the_best) {
/* p by the stored (three-decimal) score: the judged bands, monotone. */
TEST(sem_calibrated_p_bands) {
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.556F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.777F), 0.556F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.778F), 0.640F, 0.0);
- /* the cut sits at the stored precision (0.778), not the raw quartile
- * 0.777752: a stored "0.778" and a stored "0.777" never share a band */
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7779F), 0.556F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.805F), 0.640F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.806F), 0.731F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.852F), 0.857F, 0.0);
- ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.857F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.750F), 0.626F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.772F), 0.626F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.773F), 0.760F, 0.0);
+ /* the cut sits at the stored precision (0.773), not the raw quartile:
+ * a stored "0.773" and a stored "0.772" never share a band */
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.7729F), 0.626F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(0.838F), 0.760F, 0.0);
+ ASSERT_FLOAT_EQ(cbm_sem_calibrated_p(1.000F), 0.760F, 0.0);
float last = 0.0F;
for (int milli = 750; milli <= 1000; milli++) {
float p = cbm_sem_calibrated_p((float)milli / 1000.0F);
@@ -515,6 +513,24 @@ TEST(sem_calibrated_p_bands) {
PASS();
}
+/* A properties string holding a quote is read whole, and escapes are not
+ * glued to the next word (until 2026-10 the read stopped at the first quote
+ * and kept "\n" raw, so "\nreturn" became the token "nreturn"). */
+TEST(sem_json_str_honours_escapes) {
+ static const char json[] =
+ "{\"docstring\":\"Say \\\"hi\\\" to\\nthe user \\u00e9 now\",\"bt\":\"x\"}";
+ char buf[64];
+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", buf, sizeof(buf)));
+ ASSERT_STR_EQ(buf, "Say \"hi\" to the user now");
+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "bt", buf, sizeof(buf)));
+ ASSERT_STR_EQ(buf, "x");
+ ASSERT_NULL(cbm_sem_json_str(json, "absent", buf, sizeof(buf)));
+ char small[6];
+ ASSERT_NOT_NULL(cbm_sem_json_str(json, "docstring", small, sizeof(small)));
+ ASSERT_STR_EQ(small, "Say \""); /* cut at the buffer, still terminated */
+ PASS();
+}
+
TEST(sem_corpus_add_null_doc) {
cbm_sem_corpus_t *c = cbm_sem_corpus_new();
ASSERT_NOT_NULL(c);
@@ -695,6 +711,7 @@ SUITE(semantic) {
RUN_TEST(sem_combine_weighs_signals);
RUN_TEST(sem_admit_best_first_keeps_the_best);
RUN_TEST(sem_calibrated_p_bands);
+ RUN_TEST(sem_json_str_honours_escapes);
RUN_TEST(sem_corpus_add_null_doc);
RUN_TEST(sem_corpus_free_null);
RUN_TEST(sem_get_config_defaults);