Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(server): log stray errors and keep suggestions when a refresh fails (#1785)

Cihan45cbe8ccb5c1
diff --git a/server/src/lib/process-errors.js b/server/src/lib/process-errors.js
new file mode 100644
index 000000000..1bfa19cca
--- /dev/null
+++ b/server/src/lib/process-errors.js
@@ -0,0 +1,27 @@
+/**
+ * What the server does with an error nothing caught (2026-10-29).
+ *
+ * On Node 20 a promise rejection nobody handled ends the whole process, so
+ * one missed `.catch` in a background sweep took the API down for every
+ * request until pm2 restarted it. It is logged now, and the server keeps
+ * serving. An uncaught exception still ends the process, as before (after a
+ * throw nothing says its state is sound), only logged first so the crash
+ * says why; pm2 restarts it.
+ */
+
+import logger from './logger.js';
+
+export function logUnhandledRejection(reason) {
+ logger.error({ err: reason }, 'unhandled promise rejection');
+}
+
+export function exitOnUncaughtException(err, origin) {
+ logger.fatal({ err, origin }, 'uncaught exception');
+ process.exit(1);
+}
+
+/** Puts both on the process, once, when the server starts. */
+export function handleProcessErrors(proc = process) {
+ proc.on('unhandledRejection', logUnhandledRejection);
+ proc.on('uncaughtException', exitOnUncaughtException);
+}
diff --git a/server/src/lib/process-errors.test.js b/server/src/lib/process-errors.test.js
new file mode 100644
index 000000000..dd2afbdc2
--- /dev/null
+++ b/server/src/lib/process-errors.test.js
@@ -0,0 +1,97 @@
+import { EventEmitter } from 'node:events';
+import { execFile } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * An error nothing caught: a rejection is logged and the server keeps
+ * serving; an exception is logged and still ends the process for pm2.
+ */
+
+const logger = { error: vi.fn(), fatal: vi.fn() };
+vi.mock('./logger.js', () => ({ default: logger, logger }));
+
+const { handleProcessErrors } = await import('./process-errors.js');
+
+let exit;
+beforeEach(() => {
+ vi.clearAllMocks();
+ exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined);
+});
+afterEach(() => {
+ exit.mockRestore();
+});
+
+describe('handleProcessErrors', () => {
+ it('logs a rejection nobody handled, and keeps the process running', () => {
+ const proc = new EventEmitter();
+ handleProcessErrors(proc);
+ const reason = new Error('sweep failed');
+
+ proc.emit('unhandledRejection', reason, Promise.resolve());
+
+ expect(logger.error).toHaveBeenCalledWith({ err: reason }, 'unhandled promise rejection');
+ expect(exit).not.toHaveBeenCalled();
+ });
+
+ it('logs an uncaught exception, then ends the process with 1', () => {
+ const proc = new EventEmitter();
+ handleProcessErrors(proc);
+ const err = new TypeError('x is undefined');
+
+ proc.emit('uncaughtException', err, 'uncaughtException');
+
+ expect(logger.fatal).toHaveBeenCalledWith(
+ { err, origin: 'uncaughtException' },
+ 'uncaught exception',
+ );
+ expect(exit).toHaveBeenCalledWith(1);
+ expect(logger.fatal.mock.invocationCallOrder[0]).toBeLessThan(exit.mock.invocationCallOrder[0]);
+ });
+});
+
+describe('in a real process', () => {
+ const run = (code) =>
+ promisify(execFile)(process.execPath, ['--input-type=module', '-e', code], {
+ cwd: fileURLToPath(new URL('../..', import.meta.url)),
+ });
+ const moduleUrl = new URL('./process-errors.js', import.meta.url).href;
+
+ it('outlives a rejection nobody handled, and logs it', async () => {
+ const { stdout } = await run(`
+ import { handleProcessErrors } from '${moduleUrl}';
+ handleProcessErrors();
+ Promise.reject(new Error('sweep failed'));
+ setTimeout(() => console.log('still serving'), 50);
+ `);
+ expect(stdout).toContain('unhandled promise rejection');
+ expect(stdout).toContain('sweep failed');
+ expect(stdout).toContain('still serving');
+ });
+
+ it('logs an uncaught exception and ends with 1', async () => {
+ const failed = await run(`
+ import { handleProcessErrors } from '${moduleUrl}';
+ handleProcessErrors();
+ setTimeout(() => { throw new Error('boom'); }, 0);
+ setTimeout(() => console.log('still serving'), 50);
+ `).catch((err) => err);
+ expect(failed.code).toBe(1);
+ expect(failed.stdout).toContain('uncaught exception');
+ expect(failed.stdout).toContain('boom');
+ expect(failed.stdout).not.toContain('still serving');
+ });
+});
+
+/** server.js can't be imported here (it listens and starts the crons), so it is read. */
+describe("server.js's start", () => {
+ it('puts the handlers on the process before the app is made', async () => {
+ const source = await readFile(new URL('../server.js', import.meta.url), 'utf8');
+ expect(source).toMatch(/import \{ handleProcessErrors \} from '\.\/lib\/process-errors\.js';/);
+ const handled = source.indexOf('\nhandleProcessErrors();');
+ expect(handled).toBeGreaterThan(0);
+ expect(handled).toBeLessThan(source.indexOf('const app = express();'));
+ });
+});
diff --git a/server/src/routes/prompts-suggestions-refresh.test.js b/server/src/routes/prompts-suggestions-refresh.test.js
index 2fa08279d..bbecfcaf4 100644
--- a/server/src/routes/prompts-suggestions-refresh.test.js
+++ b/server/src/routes/prompts-suggestions-refresh.test.js
@@ -17,12 +17,24 @@ let db;
function table(name) {
let op = 'select';
let row = null;
+ const filters = {};
const result = (single = false) => {
if (op === 'insert') {
db.inserts.push({ table: name, row });
- return { data: name === 'prompt_suggestions' ? row : null, error: null };
+ if (name !== 'prompt_suggestions') return { data: null, error: null };
+ if (db.insertError) return { data: null, error: db.insertError };
+ return { data: row.map((r, i) => ({ id: `new-${i}`, ...r })), error: null };
}
- if (op === 'delete') return { data: null, error: null };
+ if (op === 'update') {
+ db.updates.push({ table: name, row, filters });
+ return { data: { ...row, id: filters.id }, error: null };
+ }
+ if (op === 'delete') {
+ db.deletes.push({ table: name, filters });
+ return { data: null, error: null };
+ }
+ // The undecided batch the refresh replaces.
+ if (name === 'prompt_suggestions') return { data: db.current, error: null };
if (name === 'brands') {
const brand = { id: 'b1', name: 'Acme', language: 'en', organization_id: 'o1' };
return { data: brand, error: null };
@@ -53,6 +65,19 @@ function table(name) {
return chain;
};
}
+ if (prop === 'update') {
+ return (r) => {
+ op = 'update';
+ row = r;
+ return chain;
+ };
+ }
+ if (prop === 'eq' || prop === 'in') {
+ return (column, value) => {
+ filters[column] = value;
+ return chain;
+ };
+ }
return () => chain;
},
},
@@ -114,7 +139,7 @@ const modelEnv = process.env.PROMPT_SUGGESTION_MODEL;
beforeEach(async () => {
vi.clearAllMocks();
delete process.env.PROMPT_SUGGESTION_MODEL;
- db = { inserts: [] };
+ db = { inserts: [], updates: [], deletes: [], current: [], insertError: null };
reserveRelevanceAllowance.mockResolvedValue({ unlimited: false, id: 'r1' });
getCatalogSuggestionCandidates.mockResolvedValue([]);
settleRelevanceUsage.mockResolvedValue();
@@ -124,7 +149,7 @@ beforeEach(async () => {
app.use(express.json());
app.use((req, _res, next) => {
req.user = { id: 'u1' };
- req.log = { error: vi.fn(), info: vi.fn() };
+ req.log = { error: vi.fn(), info: vi.fn(), warn: vi.fn() };
next();
});
app.use('/api/prompts', router);
@@ -268,3 +293,54 @@ describe('suggestion refresh from the catalog (ECOM-003)', () => {
expect(prompt).not.toContain('product catalog categories');
});
});
+
+describe('suggestion refresh keeps the list', () => {
+ const old = (id, text) => ({ id, brand_id: 'b1', suggested_text: text, status: 'new' });
+ const deletedIds = () =>
+ db.deletes.filter((d) => d.table === 'prompt_suggestions').flatMap((d) => d.filters.id);
+
+ it('adds the new batch first, then removes the old one', async () => {
+ db.current = [old('o1', 'an old question'), old('o2', 'another old question')];
+
+ const res = await refresh();
+
+ expect(res.status).toBe(200);
+ const [rows] = insertsInto('prompt_suggestions');
+ expect(rows).toHaveLength(15);
+ expect(deletedIds()).toEqual(['o1', 'o2']);
+ expect(db.deletes[0].filters).toMatchObject({ status: 'new' });
+ const { suggestions } = await res.json();
+ expect(suggestions).toHaveLength(15);
+ });
+
+ it('leaves the old batch in place when the insert fails', async () => {
+ db.current = [old('o1', 'an old question')];
+ // A database without 00362 turns down source 'catalog'.
+ db.insertError = { code: '23514', message: 'violates check constraint' };
+
+ expect((await refresh()).status).toBe(500);
+ expect(db.deletes).toEqual([]);
+ expect(db.updates).toEqual([]);
+ });
+
+ it('refreshes a text it already has in its own row, not a second one', async () => {
+ const repeated = suggestion(3).text;
+ db.current = [old('o1', repeated.toUpperCase()), old('o2', 'an old question')];
+
+ const res = await refresh();
+
+ expect(res.status).toBe(200);
+ const [rows] = insertsInto('prompt_suggestions');
+ expect(rows).toHaveLength(14);
+ expect(rows.some((r) => r.suggested_text === repeated)).toBe(false);
+ expect(db.updates).toHaveLength(1);
+ expect(db.updates[0]).toMatchObject({
+ row: { suggested_text: repeated, status: 'new' },
+ filters: { id: 'o1', status: 'new' },
+ });
+ expect(deletedIds()).toEqual(['o2']);
+ const { suggestions } = await res.json();
+ expect(suggestions).toHaveLength(15);
+ expect(suggestions[3]).toMatchObject({ id: 'o1', suggested_text: repeated });
+ });
+});
diff --git a/server/src/routes/prompts.js b/server/src/routes/prompts.js
index 32b5b68e3..62a4c9bd4 100644
--- a/server/src/routes/prompts.js
+++ b/server/src/routes/prompts.js
@@ -468,6 +468,67 @@ async function findOrCreateTopic(brandId, topicName) {
return created?.id ?? null;
}
+/**
+ * Puts a refreshed batch in place of the brand's undecided suggestions
+ * (status 'new'), in the batch's order. The new rows go in first and the old
+ * ones are removed only then: a failed insert (a database without 00362
+ * turning down source 'catalog', say) leaves the list as it was rather than
+ * empty. A text the old batch already has is refreshed in its row, since a
+ * brand keeps one undecided suggestion per text
+ * (idx_prompt_suggestions_brand_text_active).
+ */
+async function replaceNewSuggestions(brandId, rows, log) {
+ const { data: current, error: currentError } = await supabaseAdmin
+ .from('prompt_suggestions')
+ .select('*')
+ .eq('brand_id', brandId)
+ .eq('status', 'new');
+ if (currentError) throw currentError;
+ const textKey = (text) => text.toLowerCase();
+ const currentByText = new Map((current ?? []).map((row) => [textKey(row.suggested_text), row]));
+
+ const fresh = rows.filter((row) => !currentByText.has(textKey(row.suggested_text)));
+ let inserted = [];
+ if (fresh.length) {
+ const { data, error } = await supabaseAdmin.from('prompt_suggestions').insert(fresh).select();
+ if (error) throw error;
+ inserted = data ?? [];
+ }
+ const byText = new Map(inserted.map((row) => [textKey(row.suggested_text), row]));
+
+ // A repeated text takes the new reason, volume and expiry; its old row
+ // stays as it was if that update fails, and is left out once decided.
+ const generatedAt = new Date().toISOString();
+ for (const row of rows) {
+ const old = currentByText.get(textKey(row.suggested_text));
+ if (!old || byText.has(textKey(row.suggested_text))) continue;
+ const { data, error } = await supabaseAdmin
+ .from('prompt_suggestions')
+ .update({ ...row, generated_at: generatedAt })
+ .eq('id', old.id)
+ .eq('status', 'new')
+ .select()
+ .maybeSingle();
+ if (error) log.warn({ err: error, id: old.id }, 'prompt-suggestions refresh: kept an old row');
+ const refreshed = error ? old : data;
+ if (refreshed) byText.set(textKey(row.suggested_text), refreshed);
+ }
+
+ // Only then the rest of the old batch goes; one decided meanwhile stays.
+ const kept = new Set([...byText.values()].map((row) => row.id));
+ const stale = (current ?? []).filter((row) => !kept.has(row.id)).map((row) => row.id);
+ if (stale.length) {
+ const { error } = await supabaseAdmin
+ .from('prompt_suggestions')
+ .delete()
+ .in('id', stale)
+ .eq('status', 'new');
+ if (error) log.warn({ err: error }, 'prompt-suggestions refresh: old batch not removed');
+ }
+
+ return [...new Set(rows.map((row) => byText.get(textKey(row.suggested_text))).filter(Boolean))];
+}
+
/** Generating suggestions and Improve sets costs AI: admins and managers start it (2026-10-29). */
const SUGGEST_PROMPTS_REFUSAL = 'Only admins and managers can generate prompt suggestions.';
const IMPROVE_REFUSAL = 'Only admins and managers can use Improve.';
@@ -485,12 +546,6 @@ router.post(
log: req.log,
});
- await supabaseAdmin
- .from('prompt_suggestions')
- .delete()
- .eq('brand_id', req.params.brandId)
- .eq('status', 'new');
-
const expiresAt = new Date(Date.now() + SUGGESTION_TTL_HOURS * 60 * 60 * 1000).toISOString();
// Provenance: a suggestion is 'gsc' only when its sourceQuery matches a
@@ -572,13 +627,8 @@ router.post(
});
}
- const { data: inserted, error } = await supabaseAdmin
- .from('prompt_suggestions')
- .insert(rows)
- .select();
- if (error) throw error;
-
- return res.json({ suggestions: inserted });
+ const suggestionsNow = await replaceNewSuggestions(req.params.brandId, rows, req.log);
+ return res.json({ suggestions: suggestionsNow });
} catch (error) {
const status = error.status || 500;
req.log.error({ err: error }, 'prompt-suggestions refresh error');
diff --git a/server/src/routes/topics-suggestions-refresh.test.js b/server/src/routes/topics-suggestions-refresh.test.js
new file mode 100644
index 000000000..984416bfe
--- /dev/null
+++ b/server/src/routes/topics-suggestions-refresh.test.js
@@ -0,0 +1,142 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+
+/**
+ * POST /api/topics/suggestions/:brandId/refresh puts the new batch in before
+ * it removes the undecided one, so a failed insert leaves the list as it was
+ * rather than empty. The model is a stub.
+ */
+
+const generateObject = vi.fn();
+let db;
+
+function table(name) {
+ let op = 'select';
+ let row = null;
+ const filters = [];
+ const result = () => {
+ if (op === 'insert') {
+ db.inserts.push(row);
+ if (db.insertError) return { data: null, error: db.insertError };
+ return { data: row.map((r, i) => ({ id: `new-${i}`, ...r })), error: null };
+ }
+ if (op === 'delete') {
+ db.deletes.push(filters);
+ return { data: null, error: null };
+ }
+ if (name === 'brands') return { data: { name: 'Acme', language: 'en' }, error: null };
+ return { data: null, error: null };
+ };
+ const chain = new Proxy(
+ {},
+ {
+ get: (_t, prop) => {
+ if (prop === 'then') {
+ return (resolve, reject) => Promise.resolve(result()).then(resolve, reject);
+ }
+ if (prop === 'single' || prop === 'maybeSingle') return async () => result();
+ if (prop === 'insert') {
+ return (r) => {
+ op = 'insert';
+ row = r;
+ return chain;
+ };
+ }
+ if (prop === 'delete') {
+ return () => {
+ op = 'delete';
+ return chain;
+ };
+ }
+ if (prop === 'eq' || prop === 'not') {
+ return (...args) => {
+ filters.push([prop, ...args]);
+ return chain;
+ };
+ }
+ return () => chain;
+ },
+ },
+ );
+ return chain;
+}
+
+let server;
+let baseUrl;
+
+beforeEach(async () => {
+ vi.clearAllMocks();
+ vi.resetModules();
+ db = { inserts: [], deletes: [], insertError: null };
+ generateObject.mockResolvedValue({
+ object: { topics: [{ name: 'Coffee subscriptions' }, { name: 'Office coffee' }] },
+ });
+ vi.doMock('ai', () => ({ generateObject }));
+ vi.doMock('../lib/access.js', () => ({
+ MANAGER_ROLES: ['admin', 'manager'],
+ assertBrandAccess: vi.fn(),
+ assertRole: vi.fn(async () => 'manager'),
+ }));
+ vi.doMock('../lib/action-center/from-topic-opportunity.js', () => ({
+ sendTopicOpportunityToActionCenter: vi.fn(),
+ }));
+ vi.doMock('../lib/action-center/from-topic-gap.js', () => ({
+ sendTopicGapToActionCenter: vi.fn(),
+ }));
+ vi.doMock('../lib/plan-guard.js', () => ({ requireFeature: () => (req, res, next) => next() }));
+ vi.doMock('../lib/ai-provider.js', () => ({ resolveModel: vi.fn() }));
+ vi.doMock('../lib/web-research.js', () => ({
+ webResearch: vi.fn(async () => ({ text: 'research', searched: true, sources: [] })),
+ }));
+ vi.doMock('../lib/logger.js', () => {
+ const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), fatal: vi.fn() };
+ return { default: logger, logger };
+ });
+ vi.doMock('../config/supabase.js', () => ({ default: { from: (name) => table(name) } }));
+
+ const { default: router } = await import('./topics.js');
+ const app = express();
+ app.use(express.json());
+ app.use((req, _res, next) => {
+ req.user = { id: 'u1' };
+ req.log = { error: vi.fn(), info: vi.fn(), warn: vi.fn() };
+ next();
+ });
+ app.use('/api/topics', router);
+ await new Promise((resolve) => {
+ server = app.listen(0, resolve);
+ });
+ baseUrl = `http://127.0.0.1:${server.address().port}/api/topics`;
+});
+
+afterEach(async () => {
+ await new Promise((resolve) => server.close(resolve));
+});
+
+const refresh = () => fetch(`${baseUrl}/suggestions/b1/refresh`, { method: 'POST' });
+
+describe('POST /api/topics/suggestions/:brandId/refresh', () => {
+ it('adds the new batch, then removes the undecided rows it didn’t just add', async () => {
+ const res = await refresh();
+
+ expect(res.status).toBe(200);
+ expect(db.inserts).toHaveLength(1);
+ expect(db.inserts[0].map((r) => r.name)).toEqual(['Coffee subscriptions', 'Office coffee']);
+ expect(db.deletes).toEqual([
+ [
+ ['eq', 'brand_id', 'b1'],
+ ['eq', 'status', 'new'],
+ ['not', 'id', 'in', '(new-0,new-1)'],
+ ],
+ ]);
+ const { suggestions } = await res.json();
+ expect(suggestions.map((s) => s.id)).toEqual(['new-0', 'new-1']);
+ });
+
+ it('leaves the undecided batch in place when the insert fails', async () => {
+ db.insertError = { code: '57014', message: 'canceling statement due to statement timeout' };
+
+ expect((await refresh()).status).toBe(500);
+ expect(db.deletes).toEqual([]);
+ });
+});
diff --git a/server/src/routes/topics.js b/server/src/routes/topics.js
index 26620d509..47cecfc16 100644
--- a/server/src/routes/topics.js
+++ b/server/src/routes/topics.js
@@ -335,29 +335,36 @@ router.post(
});
// Replace the undecided batch wholesale — same behavior as prompt
- // suggestions refresh. Dismissed/added rows are never touched.
- await supabaseAdmin
+ // suggestions refresh. Dismissed/added rows are never touched. The new
+ // rows go in first and the old ones are removed only then, so a failed
+ // insert leaves the list as it was rather than empty.
+ let inserted = [];
+ if (fresh.length) {
+ const { data, error } = await supabaseAdmin
+ .from('topic_suggestions')
+ .insert(
+ fresh.map((t) => ({
+ brand_id: brandId,
+ name: t.name.trim(),
+ source: 'llm',
+ status: 'new',
+ })),
+ )
+ .select();
+ if (error) throw error;
+ inserted = data ?? [];
+ }
+
+ let stale = supabaseAdmin
.from('topic_suggestions')
.delete()
.eq('brand_id', brandId)
.eq('status', 'new');
-
- if (fresh.length === 0) {
- return res.json({ suggestions: [] });
- }
-
- const { data: inserted, error } = await supabaseAdmin
- .from('topic_suggestions')
- .insert(
- fresh.map((t) => ({
- brand_id: brandId,
- name: t.name.trim(),
- source: 'llm',
- status: 'new',
- })),
- )
- .select();
- if (error) throw error;
+ if (inserted.length)
+ stale = stale.not('id', 'in', `(${inserted.map((t) => t.id).join(',')})`);
+ const { error: staleError } = await stale;
+ if (staleError)
+ req.log.warn({ err: staleError }, 'topic-suggestions refresh: old batch not removed');
return res.json({ suggestions: inserted });
} catch (error) {
diff --git a/server/src/server.js b/server/src/server.js
index e3747d774..fac563443 100644
--- a/server/src/server.js
+++ b/server/src/server.js
@@ -9,6 +9,7 @@ import { Server as SocketIOServer } from 'socket.io';
import cron from 'node-cron';
import logger from './lib/logger.js';
+import { handleProcessErrors } from './lib/process-errors.js';
import Middleware from './middleware/index.js';
import { apiLimiter } from './middleware/rate-limiter.js';
import requestIdMiddleware from './middleware/request-id.js';
@@ -76,6 +77,9 @@ import { refreshCloroCountries } from './lib/cloro-countries.js';
import { chatAgenticDeployed } from './lib/chat-flag.js';
import { startModelPriceSync } from './lib/model-prices.js';
+// Before anything starts: a stray rejection is logged, not a crash.
+handleProcessErrors();
+
const app = express();
app.set('trust proxy', 1);
const server = http.createServer(app);