Skip to content

World's first open-source enterprise world model.

Keep empty SQL fields out of numeric benchmark scores (#1102)

Virtue0394c8836f6a669
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 87890571..cb6c2a72 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -178,5 +178,8 @@ jobs:
run: pnpm install --frozen-lockfile
- name: Test
run: pnpm test
+ # 测量台的判等不依赖任何包,借这个 job 现成的 node 跑,不另开一条流水线
+ - name: Bench scoring tests
+ run: node --test ../scripts/bench/lib.test.mjs
- name: Build (含类型检查)
run: pnpm build
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 0006aa9c..1331d691 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -252,6 +252,16 @@ node scripts/bench/ask.mjs --kb <id> --recall 8 # 只量检索:那条对
`lib.mjs` 是两个测量台共用的地基——**判等必须是同一份**,各写一份 `same()`
迟早漂移,而一旦漂移,「提议对了几条」与「答案对了几条」就不是同一把尺子量出来的。
+SQL 的空字段(psql 默认显示的 NULL、空字符串)与纯空白不参与数值判等,不能当成 `0`。
+`firstRow()` 仍取第一行各列的数;`value()` 只读第一列,第一列为空就返回空,不取后面的数。
+回归测试不需要模型服务;第二条额外验证真实 PostgreSQL 的输出,只执行 SELECT:
+
+```bash
+node --test scripts/bench/lib.test.mjs
+BENCH_TEST_PSQL="psql -X -h 127.0.0.1 -p 1517 -U utopia -tAc" BENCH_TEST_DB=utopia PGPASSWORD=utopia \
+ node --test scripts/bench/lib.pg.test.mjs
+```
+
## 本体的两个数(0061 决定 5)
`competency.mjs` 把一个库**接受了的**能力问题按人在 chat 里问的方式问一遍(走 `/kbs/{id}/chat`,
diff --git a/scripts/bench/lib.mjs b/scripts/bench/lib.mjs
index 1e46c8be..5c176a3a 100644
--- a/scripts/bench/lib.mjs
+++ b/scripts/bench/lib.mjs
@@ -70,6 +70,10 @@ export const psql = (sql) => run(withDb(PSQL, APP_DB), sql);
export const onDb = (db, sql) => run(withDb(PSQL, db), sql);
export const num = (sql) => Number(psql(sql) || 0);
+// psql 默认把 NULL 与空字符串都印成空字段;Number("") 却是 0,
+// 让没有数的字段命中零值真值。两条读数路径都先排除空白,真实的 0 照常读。
+const numericField = (field) => field.trim() === "" ? undefined : Number(field);
+
/// 一条 SQL 第一行里的所有数字。
///
/// **模型不写单列查询。** 问「平均行金额」,它跑的是
@@ -81,7 +85,7 @@ export function firstRow(db, sql) {
const out = onDb(db, `SET statement_timeout = '20s'; ${sql}`);
const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0];
if (first === undefined) return { empty: true };
- const ns = first.split("|").map((x) => Number(x)).filter((n) => Number.isFinite(n));
+ const ns = first.split("|").map(numericField).filter((n) => Number.isFinite(n));
return { ns };
} catch (e) {
return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) };
@@ -97,7 +101,8 @@ export function value(db, sql) {
// 而 -tA 下数据行不带标签——不滤掉它,每条读到的第一行都是 `SET`
const first = out.split("\n").map((l) => l.trim()).filter((l) => l !== "" && l !== "SET")[0];
if (first === undefined) return { empty: true };
- return { n: Number(String(first).split("|")[0]) };
+ const n = numericField(first.split("|")[0]);
+ return n === undefined ? { empty: true } : { n };
} catch (e) {
return { error: String(e.stderr || e.message).split("\n").filter((l) => l.trim())[0]?.slice(0, 120) };
}
diff --git a/scripts/bench/lib.pg.test.mjs b/scripts/bench/lib.pg.test.mjs
new file mode 100644
index 00000000..911b9452
--- /dev/null
+++ b/scripts/bench/lib.pg.test.mjs
@@ -0,0 +1,24 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+// BENCH_TEST_PSQL is a psql command with -tAc and connection options; BENCH_TEST_DB
+// names an existing test database. These probes only SELECT, with no fixture writes.
+test("real psql keeps blank SQL fields out of numeric benchmark scores", {
+ skip: !process.env.BENCH_TEST_PSQL,
+}, async () => {
+ assert.ok(process.env.BENCH_TEST_DB, "BENCH_TEST_DB must name the test database");
+ process.env.BENCH_PSQL = process.env.BENCH_TEST_PSQL;
+ const { firstRow, value, roughly } = await import("./lib.mjs");
+ const db = process.env.BENCH_TEST_DB;
+ for (const field of ["NULL::numeric", "''::text", "' '::text"]) {
+ const sql = `SELECT 7, ${field}`;
+ const { ns } = firstRow(db, sql);
+ assert.deepEqual(ns, [7], sql);
+ assert.equal(ns.some((n) => roughly(n, 0)), false, sql);
+ assert.deepEqual(value(db, `SELECT ${field}, 7`), { empty: true }, field);
+ }
+ assert.deepEqual(firstRow(db, "SELECT 7, 0, -2.5, 0.25"), { ns: [7, 0, -2.5, 0.25] });
+ assert.deepEqual(value(db, "SELECT 0"), { n: 0 });
+ assert.deepEqual(firstRow(db, "SELECT NULL::numeric"), { empty: true });
+ assert.deepEqual(firstRow(db, "SELECT 0 WHERE FALSE"), { empty: true });
+});
diff --git a/scripts/bench/lib.test.mjs b/scripts/bench/lib.test.mjs
new file mode 100644
index 00000000..b9fc6659
--- /dev/null
+++ b/scripts/bench/lib.test.mjs
@@ -0,0 +1,86 @@
+import assert from "node:assert/strict";
+import childProcess from "node:child_process";
+import { syncBuiltinESMExports } from "node:module";
+import test from "node:test";
+import { firstRow, value, same, roughly } from "./lib.mjs";
+
+// Stub only psql's stdout: the production command wrapper, parsers and comparison
+// functions still run, without needing a database for these boundary cases.
+function stdout(t, text) {
+ t.mock.method(childProcess, "execFileSync", () => text);
+ syncBuiltinESMExports();
+ t.after(() => {
+ t.mock.restoreAll();
+ syncBuiltinESMExports();
+ });
+}
+
+for (const [label, output] of [
+ ["trailing NULL or empty string", "SET\n7|\n"],
+ ["leading NULL or empty string", "SET\n|7\n"],
+ ["interior empty field", "SET\n7||9\n"],
+ ["whitespace-only field", "SET\n7| \t \n"],
+]) {
+ test(`firstRow does not let ${label} match a zero answer`, (t) => {
+ stdout(t, output);
+ const ns = firstRow("fixture", "SELECT fixture").ns;
+ assert.deepEqual(ns, label === "interior empty field" ? [7, 9] : [7]);
+ // ask.mjs searches every returned number using roughly; mapping scores use same.
+ assert.equal(ns.some((n) => roughly(n, 0)), false);
+ assert.equal(ns.some((n) => same(n, 0)), false);
+ });
+}
+
+test("firstRow keeps real zero, negatives, decimals and numbers in later columns", (t) => {
+ stdout(t, "SET\ncontext|0|-7|0.25|-2.5|Infinity|NaN\n999\n");
+ const { ns } = firstRow("fixture", "SELECT fixture");
+ assert.deepEqual(ns, [0, -7, 0.25, -2.5]);
+ assert.equal(ns.some((n) => roughly(n, 0)), true);
+});
+
+test("a first row of empty fields contains no numbers", (t) => {
+ stdout(t, "SET\n| |\n");
+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { ns: [] });
+});
+
+for (const output of ["SET\n", "SET\n\n"]) {
+ test(`no rows or a single blank value stays empty (${JSON.stringify(output)})`, (t) => {
+ stdout(t, output);
+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { empty: true });
+ assert.deepEqual(value("fixture", "SELECT fixture"), { empty: true });
+ });
+}
+
+for (const [label, output] of [
+ ["NULL or empty", "SET\n|7\n"],
+ ["whitespace-only", "SET\n \t |7\n"],
+]) {
+ test(`value keeps a ${label} first column empty instead of selecting a later number`, (t) => {
+ stdout(t, output);
+ const got = value("fixture", "SELECT fixture");
+ assert.deepEqual(got, { empty: true });
+ assert.equal(same(got.n, 0), false);
+ assert.equal(roughly(got.n, 0), false);
+ });
+}
+
+for (const n of [0, -7, 0.25]) {
+ test(`value preserves a numeric first column (${n})`, (t) => {
+ stdout(t, `SET\n${n}|999\n`);
+ assert.deepEqual(value("fixture", "SELECT fixture"), { n });
+ });
+}
+
+test("a nonnumeric first column does not become a later numeric value", (t) => {
+ stdout(t, "SET\ntext|0\n");
+ assert.ok(Number.isNaN(value("fixture", "SELECT fixture").n));
+});
+
+test("SQL errors retain their diagnostic instead of supplying a number", (t) => {
+ stdout(t, "");
+ childProcess.execFileSync.mock.mockImplementation(() => {
+ throw Object.assign(new Error("command failed"), { stderr: "ERROR: bad SQL\n" });
+ });
+ assert.deepEqual(firstRow("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" });
+ assert.deepEqual(value("fixture", "SELECT fixture"), { error: "ERROR: bad SQL" });
+});