Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Edge-whitespace SQL stays ASCII so it parses on non-UTF8 databases

garrytanf732a6545a6e
diff --git a/src/core/edge-whitespace.ts b/src/core/edge-whitespace.ts
index 891bff8630..56709776f4 100644
--- a/src/core/edge-whitespace.ts
+++ b/src/core/edge-whitespace.ts
@@ -6,9 +6,16 @@
*/
export const EDGE_WHITESPACE = ' \t\n\v\f\r\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff';
-const EDGE_WHITESPACE_SQL = `E'${[...EDGE_WHITESPACE].map(c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`).join('')}'`;
+const ASCII_EDGES = [...EDGE_WHITESPACE].filter(c => c.charCodeAt(0) < 0x80);
+const WIDE_EDGES_HEX = Buffer.from([...EDGE_WHITESPACE].filter(c => c.charCodeAt(0) >= 0x80).join(''), 'utf8').toString('hex');
+const EDGE_WHITESPACE_SQL = `(E'${ASCII_EDGES.map(c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`).join('')}'`
+ + ` || CASE WHEN current_setting('server_encoding') = 'UTF8' THEN convert_from(decode('${WIDE_EDGES_HEX}', 'hex'), 'UTF8') ELSE '' END)`;
-/** `btrim` of a code-provided SQL expression over EDGE_WHITESPACE: the SQL twin of `.trim()`. */
+/**
+ * `btrim` of a code-provided SQL expression over EDGE_WHITESPACE: the SQL twin of `.trim()`. The SQL text stays ASCII,
+ * since Postgres refuses a `\u` escape above U+007F on a database whose encoding is not UTF8; there it trims the ASCII
+ * edge characters only.
+ */
export function edgeTrimSql(expr: string): string {
return `btrim(${expr}, ${EDGE_WHITESPACE_SQL})`;
}
diff --git a/test/edge-whitespace-twins.test.ts b/test/edge-whitespace-twins.test.ts
index c3fccc634b..0f321cf3ef 100644
--- a/test/edge-whitespace-twins.test.ts
+++ b/test/edge-whitespace-twins.test.ts
@@ -13,6 +13,7 @@
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
+import postgres from '#postgres';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { EDGE_WHITESPACE, edgeTrimSql } from '../src/core/edge-whitespace.ts';
@@ -27,6 +28,7 @@ import {
PRIVACY_ALIASES, __resetPrivateVisibilityCacheForTests, isPrivacyPrivatePage, privacyAliasSql,
privatePagesFilterFastForCaller, privatePagesFilterForCaller, privatePagesFilterFragment, privatePagesFilterFragmentFast,
} from '../src/core/search/private-visibility.ts';
+import { assertSafeE2eDatabaseUrl } from './helpers/db-guard.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
@@ -62,6 +64,34 @@ test('EDGE_WHITESPACE is exactly what String.prototype.trim strips', () => {
expect(EDGES.sort()).toEqual(stripped.sort());
});
+test('edgeTrimSql is ASCII SQL with no Unicode escape above U+007F', () => {
+ const sql = edgeTrimSql('v');
+ expect(/^[\x20-\x7e]*$/.test(sql)).toBe(true);
+ expect(sql.match(/\\u[0-9a-f]{4}/gi)?.filter(e => parseInt(e.slice(2), 16) > 0x7f) ?? []).toEqual([]);
+});
+
+test.skipIf(!process.env.DATABASE_URL)('edgeTrimSql runs on SQL_ASCII and LATIN1 databases and trims their ASCII edges', async () => {
+ assertSafeE2eDatabaseUrl(process.env.DATABASE_URL!);
+ const admin = postgres(process.env.DATABASE_URL!, { max: 1, prepare: false });
+ try {
+ for (const encoding of ['SQL_ASCII', 'LATIN1']) {
+ const name = `gbrain_test_edges_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
+ await admin.unsafe(`CREATE DATABASE ${name} TEMPLATE template0 ENCODING '${encoding}' LC_COLLATE 'C' LC_CTYPE 'C'`);
+ const url = new URL(process.env.DATABASE_URL!); url.pathname = `/${name}`;
+ const db = postgres(url.toString(), { max: 1, prepare: false });
+ try {
+ const [row] = await db.unsafe(`SELECT current_setting('server_encoding') AS encoding, ${edgeTrimSql('v')} AS t FROM (VALUES ($1::text)) AS x(v)`, ['\t a b \n\r']);
+ expect({ encoding: row!.encoding, t: row!.t }).toEqual({ encoding, t: 'a b' });
+ } finally {
+ await db.end();
+ await admin.unsafe(`DROP DATABASE ${name} WITH (FORCE)`);
+ }
+ }
+ } finally {
+ await admin.end();
+ }
+}, 60_000);
+
describe.each(testBackends())('%s', (backend) => {
const engine = () => engines.find(e => e.name === backend)!.engine;