Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(news-knowledge): repair split emoji and drop NUL before writing a story so Postgres stops rejecting it
nirholas234a06a24b83
diff --git a/api/_lib/news-knowledge-store.js b/api/_lib/news-knowledge-store.jsindex e890eab49..b7b225fb5 100644--- a/api/_lib/news-knowledge-store.js+++ b/api/_lib/news-knowledge-store.js@@ -64,6 +64,21 @@ async function ensureTable() {return _ensured;}+// Postgres refuses two things JavaScript strings carry happily: an unpaired+// UTF-16 surrogate (jsonb: "invalid input syntax for type json", Unicode low+// surrogate must follow a high surrogate) and NUL (\u0000 cannot be converted+// to text). Publisher pages and length-capped slices produce both, and either+// one dropped the whole story: 120 failed writes a day in production on+// 2026-10-09. Repair surrogates to U+FFFD and drop NULs at this boundary.+export function pgSafeText(v) {+ return typeof v === 'string' ? v.toWellFormed().replaceAll('\u0000', '') : v;+}++/** JSON for a ::jsonb parameter with every string made Postgres-safe. */+export function pgSafeJson(value) {+ return JSON.stringify(value, (_key, v) => pgSafeText(v));+}+const iso = (v) => {const t = Date.parse(v || '');return Number.isNaN(t) ? null : new Date(t).toISOString();@@ -80,8 +95,9 @@ const iso = (v) => {export async function recordExtraction(record) {if (!record?.id || !record?.url || !record?.title) return false;if (!(await ensureTable())) return false;- const tickers = Array.isArray(record.tickers) ? record.tickers.slice(0, 12) : [];- const entities = Array.isArray(record.entities) ? record.entities.slice(0, 24) : [];+ const tickers = Array.isArray(record.tickers) ? record.tickers.slice(0, 12).map(pgSafeText) : [];+ const entities = Array.isArray(record.entities) ? record.entities.slice(0, 24).map(pgSafeText) : [];+ const text = (v) => pgSafeText(v || null);const doc = {paragraphs: Array.isArray(record.paragraphs) ? record.paragraphs.slice(0, 60) : [],key_points: Array.isArray(record.key_points) ? record.key_points.slice(0, 6) : [],@@ -97,12 +113,12 @@ export async function recordExtraction(record) {(id, url, title, source, author, image, published_at, extraction,sentiment, analysis, tickers, entities, summary, content_chars, doc, updated_at)values- (${record.id}, ${record.url}, ${record.title.slice(0, 500)}, ${record.source || null},- ${record.author || null}, ${record.image || null}, ${iso(record.published_at)},+ (${record.id}, ${text(record.url)}, ${text(record.title.slice(0, 500))}, ${text(record.source)},+ ${text(record.author)}, ${text(record.image)}, ${iso(record.published_at)},${record.extraction || 'preview'}, ${record.sentiment || null},${record.analysis_provider || null}, ${tickers}::text[], ${entities}::text[],- ${(record.summary || '').slice(0, 2000) || null}, ${record.content_chars || 0},- ${JSON.stringify(doc)}::jsonb, now())+ ${text((record.summary || '').slice(0, 2000))}, ${record.content_chars || 0},+ ${pgSafeJson(doc)}::jsonb, now())on conflict (id) do update seturl = excluded.url,title = excluded.title,diff --git a/tests/news-knowledge-store-pg-safe.test.js b/tests/news-knowledge-store-pg-safe.test.jsnew file mode 100644index 000000000..c2f6dc01a--- /dev/null+++ b/tests/news-knowledge-store-pg-safe.test.js@@ -0,0 +1,56 @@+// The news knowledge store writes every extracted story to Postgres. Postgres+// refuses two things a JavaScript string carries without complaint: an unpaired+// UTF-16 surrogate inside jsonb ("invalid input syntax for type json") and NUL+// ("\u0000 cannot be converted to text"). Publisher pages and length-capped+// slices produce both, and either one dropped the whole story: 120 failed+// writes a day in production (measured 2026-10-09). The write must repair them.++import { describe, it, expect, vi } from 'vitest';++const calls = [];+vi.mock('../api/_lib/db.js', () => ({+ sql: vi.fn(async (strings, ...values) => {+ calls.push({ text: strings.join('?'), values });+ return [];+ }),+}));+vi.mock('../api/_lib/env.js', () => ({ databaseConfigured: () => true }));++const { recordExtraction } = await import('../api/_lib/news-knowledge-store.js');++const LONE_HIGH = '\ud83d';+const hasLoneSurrogate = (s) => /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/.test(s);++describe('recordExtraction writes only what Postgres accepts', () => {+ it('repairs a sliced emoji and drops NUL in the jsonb doc and the text columns', async () => {+ const ok = await recordExtraction({+ id: 'story-1',+ url: 'https://example.com/story',+ title: `Headline cut mid-emoji ${LONE_HIGH}`,+ summary: `Summary with a NUL\u0000 and a split ${LONE_HIGH}`,+ author: 'Reporter\u0000',+ paragraphs: [`First paragraph ends on half an emoji ${LONE_HIGH}`, 'Clean paragraph 😀'],+ key_points: [`Point\u0000`],+ entities: [`Entity ${LONE_HIGH}`],+ content_chars: 120,+ });+ expect(ok).toBe(true);++ const insert = calls.find((c) => /insert into news_knowledge/.test(c.text));+ expect(insert).toBeTruthy();+ const strings = insert.values.flat().filter((v) => typeof v === 'string');+ for (const s of strings) {+ expect(hasLoneSurrogate(s)).toBe(false);+ expect(s.includes('\u0000')).toBe(false);+ // JSON.stringify escapes a lone surrogate as text; that escape is what+ // Postgres rejects inside jsonb.+ expect(/\\ud[89ab][0-9a-f]{2}(?!\\ud[c-f])/i.test(s)).toBe(false);+ expect(s.includes('\\u0000')).toBe(false);+ }++ const doc = JSON.parse(strings.find((s) => s.startsWith('{"paragraphs"')));+ expect(doc.paragraphs[0]).toBe('First paragraph ends on half an emoji �');+ expect(doc.paragraphs[1]).toBe('Clean paragraph 😀');+ expect(doc.key_points[0]).toBe('Point');+ });+});