Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
fix(audit): retry the LLM calls so transient failures don't drop signals/recommendations (#261)
Gökhan Geyikd3b2a945a951
diff --git a/server/src/lib/audit/audit.test.js b/server/src/lib/audit/audit.test.jsindex 35fcf348d..b5b0e46e3 100644--- a/server/src/lib/audit/audit.test.js+++ b/server/src/lib/audit/audit.test.js@@ -5,6 +5,7 @@ import { fleschReadingEase, classifyLinks, jsonLd, countPhrases } from './signalimport { jsonLdPresence, h1Quality } from './signals/structure.js';import { https, metaDescription } from './signals/trust.js';import { length } from './signals/content.js';+import { withRetry } from './retry.js';/** Build a minimal audit context from an HTML string (no network). */function ctxFromHtml(html, extra = {}) {@@ -88,6 +89,49 @@ describe('helpers', () => {});});+describe('withRetry', () => {+ it('returns the value on first success without retrying', async () => {+ let calls = 0;+ const out = await withRetry(+ async () => {+ calls += 1;+ return 'ok';+ },+ { attempts: 3, baseDelayMs: 0 },+ );+ expect(out).toBe('ok');+ expect(calls).toBe(1);+ });++ it('retries transient failures then succeeds', async () => {+ let calls = 0;+ const out = await withRetry(+ async () => {+ calls += 1;+ if (calls < 3) throw new Error('transient');+ return 'recovered';+ },+ { attempts: 3, baseDelayMs: 0 },+ );+ expect(out).toBe('recovered');+ expect(calls).toBe(3);+ });++ it('throws the last error after exhausting attempts', async () => {+ let calls = 0;+ await expect(+ withRetry(+ async () => {+ calls += 1;+ throw new Error('always');+ },+ { attempts: 2, baseDelayMs: 0 },+ ),+ ).rejects.toThrow('always');+ expect(calls).toBe(2);+ });+});+describe('deterministic signals', () => {it('h1-quality: pass for one well-sized H1, fail for none', () => {const good = h1Quality.evaluate(ctxFromHtml('<h1>The Best Running Shoes for Marathons</h1>'));diff --git a/server/src/lib/audit/llm-signals.js b/server/src/lib/audit/llm-signals.jsindex e5b1e64c2..46490085a 100644--- a/server/src/lib/audit/llm-signals.js+++ b/server/src/lib/audit/llm-signals.js@@ -14,6 +14,7 @@ import { generateObject } from 'ai';import { z } from 'zod';import { resolveModel } from '../ai-provider.js';import { signalsByKey } from './rubric.js';+import { withRetry } from './retry.js';const DEFAULT_MODEL = 'google/gemini-3-flash-preview';@@ -86,12 +87,16 @@ export async function evaluateLlmSignals(ctx) {const modelString = process.env.AUDIT_LLM_MODEL || DEFAULT_MODEL;try {- const { object } = await generateObject({- model: resolveModel(modelString),- schema: responseSchema,- system: SYSTEM_PROMPT,- prompt: buildPrompt(ctx),- });+ const { object } = await withRetry(+ () =>+ generateObject({+ model: resolveModel(modelString),+ schema: responseSchema,+ system: SYSTEM_PROMPT,+ prompt: buildPrompt(ctx),+ }),+ { label: 'llm-signals' },+ );return LLM_SIGNAL_KEYS.map((key) => {const v = object[key];diff --git a/server/src/lib/audit/recommendations.js b/server/src/lib/audit/recommendations.jsindex 37f511d07..35680fba2 100644--- a/server/src/lib/audit/recommendations.js+++ b/server/src/lib/audit/recommendations.js@@ -14,6 +14,7 @@ import { generateObject } from 'ai';import { z } from 'zod';import { resolveModel } from '../ai-provider.js';import { signalsByKey } from './rubric.js';+import { withRetry } from './retry.js';const DEFAULT_MODEL = 'google/gemini-3-flash-preview';@@ -101,11 +102,13 @@ export async function generateRecommendations(ctx, { results } = {}) {const pageText = ctx.text.slice(0, 6000);try {- const { object } = await generateObject({- model: resolveModel(modelString),- schema: recommendationsSchema,- system: SYSTEM_PROMPT,- prompt: `PAGE URL: ${ctx.url}+ const { object } = await withRetry(+ () =>+ generateObject({+ model: resolveModel(modelString),+ schema: recommendationsSchema,+ system: SYSTEM_PROMPT,+ prompt: `PAGE URL: ${ctx.url}Improve THIS page for its own topic/intent (infer it from the content below). Never introduce subject matter the page isn't already about.@@ -118,7 +121,9 @@ ${pageText}"""Return prioritized, page-specific recommendations.`,- });+ }),+ { label: 'recommendations' },+ );return (object.recommendations || []).filter((r) => RECOMMENDABLE_SIGNALS.has(r.signalKey))diff --git a/server/src/lib/audit/retry.js b/server/src/lib/audit/retry.jsnew file mode 100644index 000000000..1d9e382d7--- /dev/null+++ b/server/src/lib/audit/retry.js@@ -0,0 +1,31 @@+/**+ * Retry wrapper for the audit's LLM calls. Provider blips (rate limits,+ * transient 5xx, the occasional malformed structured-output response) should+ * not silently drop a whole audit's LLM verdicts or recommendations — one bad+ * round-trip is usually fixed by trying again a moment later.+ */++/**+ * @template T+ * @param {() => Promise<T>} fn+ * @param {{ attempts?: number, baseDelayMs?: number, label?: string }} [opts]+ * @returns {Promise<T>}+ */+export async function withRetry(fn, { attempts = 3, baseDelayMs = 500, label = 'llm' } = {}) {+ let lastErr;+ for (let i = 0; i < attempts; i += 1) {+ try {+ return await fn();+ } catch (err) {+ lastErr = err;+ if (i < attempts - 1) {+ const delay = baseDelayMs * 2 ** i;+ console.warn(+ `[audit] ${label} attempt ${i + 1}/${attempts} failed: ${err.message}; retrying in ${delay}ms`,+ );+ await new Promise((resolve) => setTimeout(resolve, delay));+ }+ }+ }+ throw lastErr;+}