Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(news-knowledge): repair split emoji and drop NUL before writing a story so Postgres stops rejecting it

nirholas833407d4f06c
diff --git a/api/_lib/news-knowledge-store.js b/api/_lib/news-knowledge-store.js
index e890eab49..b7b225fb5 100644
--- a/api/_lib/news-knowledge-store.js
+++ b/api/_lib/news-knowledge-store.js
@@ -64,6 +64,21 @@ async function ensureTable() {
return _ensured;
}
+// Postgres refuses two things JavaScript strings carry happily: an unpaired
+// UTF-16 surrogate (jsonb: "invalid input syntax for type json", Unicode low
+// surrogate must follow a high surrogate) and NUL (\u0000 cannot be converted
+// to text). Publisher pages and length-capped slices produce both, and either
+// one dropped the whole story: 120 failed writes a day in production on
+// 2026-10-09. Repair surrogates to U+FFFD and drop NULs at this boundary.
+export function pgSafeText(v) {
+ return typeof v === 'string' ? v.toWellFormed().replaceAll('\u0000', '') : v;
+}
+
+/** JSON for a ::jsonb parameter with every string made Postgres-safe. */
+export function pgSafeJson(value) {
+ return JSON.stringify(value, (_key, v) => pgSafeText(v));
+}
+
const iso = (v) => {
const t = Date.parse(v || '');
return Number.isNaN(t) ? null : new Date(t).toISOString();
@@ -80,8 +95,9 @@ const iso = (v) => {
export async function recordExtraction(record) {
if (!record?.id || !record?.url || !record?.title) return false;
if (!(await ensureTable())) return false;
- const tickers = Array.isArray(record.tickers) ? record.tickers.slice(0, 12) : [];
- const entities = Array.isArray(record.entities) ? record.entities.slice(0, 24) : [];
+ const tickers = Array.isArray(record.tickers) ? record.tickers.slice(0, 12).map(pgSafeText) : [];
+ const entities = Array.isArray(record.entities) ? record.entities.slice(0, 24).map(pgSafeText) : [];
+ const text = (v) => pgSafeText(v || null);
const doc = {
paragraphs: Array.isArray(record.paragraphs) ? record.paragraphs.slice(0, 60) : [],
key_points: Array.isArray(record.key_points) ? record.key_points.slice(0, 6) : [],
@@ -97,12 +113,12 @@ export async function recordExtraction(record) {
(id, url, title, source, author, image, published_at, extraction,
sentiment, analysis, tickers, entities, summary, content_chars, doc, updated_at)
values
- (${record.id}, ${record.url}, ${record.title.slice(0, 500)}, ${record.source || null},
- ${record.author || null}, ${record.image || null}, ${iso(record.published_at)},
+ (${record.id}, ${text(record.url)}, ${text(record.title.slice(0, 500))}, ${text(record.source)},
+ ${text(record.author)}, ${text(record.image)}, ${iso(record.published_at)},
${record.extraction || 'preview'}, ${record.sentiment || null},
${record.analysis_provider || null}, ${tickers}::text[], ${entities}::text[],
- ${(record.summary || '').slice(0, 2000) || null}, ${record.content_chars || 0},
- ${JSON.stringify(doc)}::jsonb, now())
+ ${text((record.summary || '').slice(0, 2000))}, ${record.content_chars || 0},
+ ${pgSafeJson(doc)}::jsonb, now())
on conflict (id) do update set
url = excluded.url,
title = excluded.title,
diff --git a/tests/news-knowledge-store-pg-safe.test.js b/tests/news-knowledge-store-pg-safe.test.js
new file mode 100644
index 000000000..c2f6dc01a
--- /dev/null
+++ b/tests/news-knowledge-store-pg-safe.test.js
@@ -0,0 +1,56 @@
+// The news knowledge store writes every extracted story to Postgres. Postgres
+// refuses two things a JavaScript string carries without complaint: an unpaired
+// UTF-16 surrogate inside jsonb ("invalid input syntax for type json") and NUL
+// ("\u0000 cannot be converted to text"). Publisher pages and length-capped
+// slices produce both, and either one dropped the whole story: 120 failed
+// writes a day in production (measured 2026-10-09). The write must repair them.
+
+import { describe, it, expect, vi } from 'vitest';
+
+const calls = [];
+vi.mock('../api/_lib/db.js', () => ({
+ sql: vi.fn(async (strings, ...values) => {
+ calls.push({ text: strings.join('?'), values });
+ return [];
+ }),
+}));
+vi.mock('../api/_lib/env.js', () => ({ databaseConfigured: () => true }));
+
+const { recordExtraction } = await import('../api/_lib/news-knowledge-store.js');
+
+const LONE_HIGH = '\ud83d';
+const hasLoneSurrogate = (s) => /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/.test(s);
+
+describe('recordExtraction writes only what Postgres accepts', () => {
+ it('repairs a sliced emoji and drops NUL in the jsonb doc and the text columns', async () => {
+ const ok = await recordExtraction({
+ id: 'story-1',
+ url: 'https://example.com/story',
+ title: `Headline cut mid-emoji ${LONE_HIGH}`,
+ summary: `Summary with a NUL\u0000 and a split ${LONE_HIGH}`,
+ author: 'Reporter\u0000',
+ paragraphs: [`First paragraph ends on half an emoji ${LONE_HIGH}`, 'Clean paragraph 😀'],
+ key_points: [`Point\u0000`],
+ entities: [`Entity ${LONE_HIGH}`],
+ content_chars: 120,
+ });
+ expect(ok).toBe(true);
+
+ const insert = calls.find((c) => /insert into news_knowledge/.test(c.text));
+ expect(insert).toBeTruthy();
+ const strings = insert.values.flat().filter((v) => typeof v === 'string');
+ for (const s of strings) {
+ expect(hasLoneSurrogate(s)).toBe(false);
+ expect(s.includes('\u0000')).toBe(false);
+ // JSON.stringify escapes a lone surrogate as text; that escape is what
+ // Postgres rejects inside jsonb.
+ expect(/\\ud[89ab][0-9a-f]{2}(?!\\ud[c-f])/i.test(s)).toBe(false);
+ expect(s.includes('\\u0000')).toBe(false);
+ }
+
+ const doc = JSON.parse(strings.find((s) => s.startsWith('{"paragraphs"')));
+ expect(doc.paragraphs[0]).toBe('First paragraph ends on half an emoji �');
+ expect(doc.paragraphs[1]).toBe('Clean paragraph 😀');
+ expect(doc.key_points[0]).toBe('Point');
+ });
+});