Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Edge-whitespace SQL stays ASCII so it parses on non-UTF8 databases
garrytanf732a6545a6e
diff --git a/src/core/edge-whitespace.ts b/src/core/edge-whitespace.tsindex 891bff8630..56709776f4 100644--- a/src/core/edge-whitespace.ts+++ b/src/core/edge-whitespace.ts@@ -6,9 +6,16 @@*/export const EDGE_WHITESPACE = ' \t\n\v\f\r\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff';-const EDGE_WHITESPACE_SQL = `E'${[...EDGE_WHITESPACE].map(c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`).join('')}'`;+const ASCII_EDGES = [...EDGE_WHITESPACE].filter(c => c.charCodeAt(0) < 0x80);+const WIDE_EDGES_HEX = Buffer.from([...EDGE_WHITESPACE].filter(c => c.charCodeAt(0) >= 0x80).join(''), 'utf8').toString('hex');+const EDGE_WHITESPACE_SQL = `(E'${ASCII_EDGES.map(c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`).join('')}'`+ + ` || CASE WHEN current_setting('server_encoding') = 'UTF8' THEN convert_from(decode('${WIDE_EDGES_HEX}', 'hex'), 'UTF8') ELSE '' END)`;-/** `btrim` of a code-provided SQL expression over EDGE_WHITESPACE: the SQL twin of `.trim()`. */+/**+ * `btrim` of a code-provided SQL expression over EDGE_WHITESPACE: the SQL twin of `.trim()`. The SQL text stays ASCII,+ * since Postgres refuses a `\u` escape above U+007F on a database whose encoding is not UTF8; there it trims the ASCII+ * edge characters only.+ */export function edgeTrimSql(expr: string): string {return `btrim(${expr}, ${EDGE_WHITESPACE_SQL})`;}diff --git a/test/edge-whitespace-twins.test.ts b/test/edge-whitespace-twins.test.tsindex c3fccc634b..0f321cf3ef 100644--- a/test/edge-whitespace-twins.test.ts+++ b/test/edge-whitespace-twins.test.ts@@ -13,6 +13,7 @@*/import { afterAll, beforeAll, describe, expect, test } from 'bun:test';import { randomUUID } from 'node:crypto';+import postgres from '#postgres';import type { BrainEngine } from '../src/core/engine.ts';import { PGLiteEngine } from '../src/core/pglite-engine.ts';import { EDGE_WHITESPACE, edgeTrimSql } from '../src/core/edge-whitespace.ts';@@ -27,6 +28,7 @@ import {PRIVACY_ALIASES, __resetPrivateVisibilityCacheForTests, isPrivacyPrivatePage, privacyAliasSql,privatePagesFilterFastForCaller, privatePagesFilterForCaller, privatePagesFilterFragment, privatePagesFilterFragmentFast,} from '../src/core/search/private-visibility.ts';+import { assertSafeE2eDatabaseUrl } from './helpers/db-guard.ts';import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';import { testBackends } from './helpers/test-backends.ts';@@ -62,6 +64,34 @@ test('EDGE_WHITESPACE is exactly what String.prototype.trim strips', () => {expect(EDGES.sort()).toEqual(stripped.sort());});+test('edgeTrimSql is ASCII SQL with no Unicode escape above U+007F', () => {+ const sql = edgeTrimSql('v');+ expect(/^[\x20-\x7e]*$/.test(sql)).toBe(true);+ expect(sql.match(/\\u[0-9a-f]{4}/gi)?.filter(e => parseInt(e.slice(2), 16) > 0x7f) ?? []).toEqual([]);+});++test.skipIf(!process.env.DATABASE_URL)('edgeTrimSql runs on SQL_ASCII and LATIN1 databases and trims their ASCII edges', async () => {+ assertSafeE2eDatabaseUrl(process.env.DATABASE_URL!);+ const admin = postgres(process.env.DATABASE_URL!, { max: 1, prepare: false });+ try {+ for (const encoding of ['SQL_ASCII', 'LATIN1']) {+ const name = `gbrain_test_edges_${randomUUID().replace(/-/g, '').slice(0, 16)}`;+ await admin.unsafe(`CREATE DATABASE ${name} TEMPLATE template0 ENCODING '${encoding}' LC_COLLATE 'C' LC_CTYPE 'C'`);+ const url = new URL(process.env.DATABASE_URL!); url.pathname = `/${name}`;+ const db = postgres(url.toString(), { max: 1, prepare: false });+ try {+ const [row] = await db.unsafe(`SELECT current_setting('server_encoding') AS encoding, ${edgeTrimSql('v')} AS t FROM (VALUES ($1::text)) AS x(v)`, ['\t a b \n\r']);+ expect({ encoding: row!.encoding, t: row!.t }).toEqual({ encoding, t: 'a b' });+ } finally {+ await db.end();+ await admin.unsafe(`DROP DATABASE ${name} WITH (FORCE)`);+ }+ }+ } finally {+ await admin.end();+ }+}, 60_000);+describe.each(testBackends())('%s', (backend) => {const engine = () => engines.find(e => e.name === backend)!.engine;