World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep empty SQL fields out of numeric benchmark scores (#1102)
Virtue0394c8836f6a669
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex 87890571..cb6c2a72 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -178,5 +178,8 @@ jobs:run: pnpm install --frozen-lockfile- name: Testrun: pnpm test+ # 测量台的判等不依赖任何包,借这个 job 现成的 node 跑,不另开一条流水线+ - name: Bench scoring tests+ run: node --test ../scripts/bench/lib.test.mjs- name: Build (含类型检查)run: pnpm builddiff --git a/scripts/bench/README.md b/scripts/bench/README.mdindex 0006aa9c..1331d691 100644--- a/scripts/bench/README.md+++ b/scripts/bench/README.md@@ -252,6 +252,16 @@ node scripts/bench/ask.mjs --kb <id> --recall 8 # 只量检索:那条对`lib.mjs` 是两个测量台共用的地基——**判等必须是同一份**,各写一份 `same()`迟早漂移,而一旦漂移,「提议对了几条」与「答案对了几条」就不是同一把尺子量出来的。+SQL 的空字段(psql 默认显示的 NULL、空字符串)与纯空白不参与数值判等,不能当成 `0`。+`firstRow()` 仍取第一行各列的数;`value()` 只读第一列,第一列为空就返回空,不取后面的数。+回归测试不需要模型服务;第二条额外验证真实 PostgreSQL 的输出,只执行 SELECT:++```bash+node --test scripts/bench/lib.test.mjs+BENCH_TEST_PSQL="psql -X -h 127.0.0.1 -p 1517 -U utopia -tAc" BENCH_TEST_DB=utopia PGPASSWORD=utopia \+ node --test scripts/bench/lib.pg.test.mjs+```+## 本体的两个数(0061 决定 5)`competency.mjs` 把一个库**接受了的**能力问题按人在 chat 里问的方式问一遍(走 `/kbs/{id}/chat`,diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjsindex 1e46c8be..5c176a3a 100644--- a/scripts/bench/lib.mjs+++ b/scripts/bench/lib.mjs@@ -70,6 +70,10 @@ export const psql = (sql) => run(withDb(PSQL, APP_DB), sql);export const onDb = (db, sql) => run(withDb(PSQL, db), sql);export const num = (sql) => Number(psql(sql) || 0);+// psql 默认把 NULL 与空字符串都印成空字段;Number("") 却是 0,+// 让没有数的字段命中零值真值。两条读数路径都先排除空白,真实的 0 照常读。+const numericField = (field) => field.trim() === "" ? undefined : Number(field);+/// 一条 SQL 第一行里的所有数字。////// **模型不写单列查询。** 问「平均行金额」,它跑的是@@ -81,7 +85,7 @@ export function firstRow(db, sql) {const out = onDb(db, `SET statement_timeout = '20s'; ${sql}`);const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0];if (first === undefined) return { empty: true };- const ns = first.split("|").map((x) => Number(x)).filter((n) => Number.isFinite(n));+ const ns = first.split("|").map(numericField).filter((n) => Number.isFinite(n));return { ns };} catch (e) {return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) };@@ -97,7 +101,8 @@ export function value(db, sql) {// 而 -tA 下数据行不带标签——不滤掉它,每条读到的第一行都是 `SET`const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0];if (first === undefined) return { empty: true };- return { n: Number(String(first).split("|")[0]) };+ const n = numericField(first.split("|")[0]);+ return n === undefined ? { empty: true } : { n };} catch (e) {return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) };}diff --git a/scripts/bench/lib.pg.test.mjs b/scripts/bench/lib.pg.test.mjsnew file mode 100644index 00000000..911b9452--- /dev/null+++ b/scripts/bench/lib.pg.test.mjs@@ -0,0 +1,24 @@+import assert from "node:assert/strict";+import test from "node:test";++// BENCH_TEST_PSQL is a psql command with -tAc and connection options; BENCH_TEST_DB+// names an existing test database. These probes only SELECT, with no fixture writes.+test("real psql keeps blank SQL fields out of numeric benchmark scores", {+ skip: !process.env.BENCH_TEST_PSQL,+}, async () => {+ assert.ok(process.env.BENCH_TEST_DB, "BENCH_TEST_DB must name the test database");+ process.env.BENCH_PSQL = process.env.BENCH_TEST_PSQL;+ const { firstRow, value, roughly } = await import("./lib.mjs");+ const db = process.env.BENCH_TEST_DB;+ for (const field of ["NULL::numeric", "''::text", "' '::text"]) {+ const sql = `SELECT 7, ${field}`;+ const { ns } = firstRow(db, sql);+ assert.deepEqual(ns, [7], sql);+ assert.equal(ns.some((n) => roughly(n, 0)), false, sql);+ assert.deepEqual(value(db, `SELECT ${field}, 7`), { empty: true }, field);+ }+ assert.deepEqual(firstRow(db, "SELECT 7, 0, -2.5, 0.25"), { ns: [7, 0, -2.5, 0.25] });+ assert.deepEqual(value(db, "SELECT 0"), { n: 0 });+ assert.deepEqual(firstRow(db, "SELECT NULL::numeric"), { empty: true });+ assert.deepEqual(firstRow(db, "SELECT 0 WHERE FALSE"), { empty: true });+});diff --git a/scripts/bench/lib.test.mjs b/scripts/bench/lib.test.mjsnew file mode 100644index 00000000..b9fc6659--- /dev/null+++ b/scripts/bench/lib.test.mjs@@ -0,0 +1,86 @@+import assert from "node:assert/strict";+import childProcess from "node:child_process";+import { syncBuiltinESMExports } from "node:module";+import test from "node:test";+import { firstRow, value, same, roughly } from "./lib.mjs";++// Stub only psql's stdout: the production command wrapper, parsers and comparison+// functions still run, without needing a database for these boundary cases.+function stdout(t, text) {+ t.mock.method(childProcess, "execFileSync", () => text);+ syncBuiltinESMExports();+ t.after(() => {+ t.mock.restoreAll();+ syncBuiltinESMExports();+ });+}++for (const [label, output] of [+ ["trailing NULL or empty string", "SET\n7|\n"],+ ["leading NULL or empty string", "SET\n|7\n"],+ ["interior empty field", "SET\n7||9\n"],+ ["whitespace-only field", "SET\n7| \t \n"],+]) {+ test(`firstRow does not let ${label} match a zero answer`, (t) => {+ stdout(t, output);+ const ns = firstRow("fixture", "SELECT fixture").ns;+ assert.deepEqual(ns, label === "interior empty field" ? [7, 9] : [7]);+ // ask.mjs searches every returned number using roughly; mapping scores use same.+ assert.equal(ns.some((n) => roughly(n, 0)), false);+ assert.equal(ns.some((n) => same(n, 0)), false);+ });+}++test("firstRow keeps real zero, negatives, decimals and numbers in later columns", (t) => {+ stdout(t, "SET\ncontext|0|-7|0.25|-2.5|Infinity|NaN\n999\n");+ const { ns } = firstRow("fixture", "SELECT fixture");+ assert.deepEqual(ns, [0, -7, 0.25, -2.5]);+ assert.equal(ns.some((n) => roughly(n, 0)), true);+});++test("a first row of empty fields contains no numbers", (t) => {+ stdout(t, "SET\n| |\n");+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { ns: [] });+});++for (const output of ["SET\n", "SET\n\n"]) {+ test(`no rows or a single blank value stays empty (${JSON.stringify(output)})`, (t) => {+ stdout(t, output);+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { empty: true });+ assert.deepEqual(value("fixture", "SELECT fixture"), { empty: true });+ });+}++for (const [label, output] of [+ ["NULL or empty", "SET\n|7\n"],+ ["whitespace-only", "SET\n \t |7\n"],+]) {+ test(`value keeps a ${label} first column empty instead of selecting a later number`, (t) => {+ stdout(t, output);+ const got = value("fixture", "SELECT fixture");+ assert.deepEqual(got, { empty: true });+ assert.equal(same(got.n, 0), false);+ assert.equal(roughly(got.n, 0), false);+ });+}++for (const n of [0, -7, 0.25]) {+ test(`value preserves a numeric first column (${n})`, (t) => {+ stdout(t, `SET\n${n}|999\n`);+ assert.deepEqual(value("fixture", "SELECT fixture"), { n });+ });+}++test("a nonnumeric first column does not become a later numeric value", (t) => {+ stdout(t, "SET\ntext|0\n");+ assert.ok(Number.isNaN(value("fixture", "SELECT fixture").n));+});++test("SQL errors retain their diagnostic instead of supplying a number", (t) => {+ stdout(t, "");+ childProcess.execFileSync.mock.mockImplementation(() => {+ throw Object.assign(new Error("command failed"), { stderr: "ERROR: bad SQL\n" });+ });+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" });+ assert.deepEqual(value("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" });+});