Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Show standard copy instead of provider text when an AI visibility answer fails (#928)
Carter Smith2249317f767a
diff --git a/src/client/features/ai-visibility/AnswerContent.tsx b/src/client/features/ai-visibility/AnswerContent.tsxindex bc30cfd2..6c6aad43 100644--- a/src/client/features/ai-visibility/AnswerContent.tsx+++ b/src/client/features/ai-visibility/AnswerContent.tsx@@ -11,6 +11,7 @@ import {AccordionTrigger,} from "@/client/components/ui/accordion";import { Badge } from "@/client/components/ui/badge";+import { getStandardErrorMessage } from "@/client/lib/error-messages";import { Button } from "@/client/components/ui/button";import {Tabs,@@ -77,7 +78,7 @@ function AnswerEvidence({ answer }: { answer: AiAnswer }) {role="alert"className="border-b bg-destructive/5 px-5 py-3 text-sm text-destructive border-border">- {answer.observation.error}+ {getStandardErrorMessage(new Error(answer.observation.error))}</p>)}{answer.truncated && (diff --git a/src/client/features/ai-visibility/PromptAnswers.tsx b/src/client/features/ai-visibility/PromptAnswers.tsxindex 8aa66021..26c40a4c 100644--- a/src/client/features/ai-visibility/PromptAnswers.tsx+++ b/src/client/features/ai-visibility/PromptAnswers.tsx@@ -5,6 +5,7 @@ import {type AiObservationRow,type AiPrompt,} from "@/shared/ai-visibility";+import { getStandardErrorMessage } from "@/client/lib/error-messages";import { AnswerContent } from "./AnswerContent";export function PromptAnswers({@@ -49,8 +50,9 @@ export function PromptAnswers({{aiObservationStatusLabel(observation.status)}</p><p className="text-sm text-muted-foreground">- {observation.error ??- "An answer is not available for this collection yet."}+ {observation.error+ ? getStandardErrorMessage(new Error(observation.error))+ : "An answer is not available for this collection yet."}</p><p className="text-xs text-muted-foreground">This collection is excluded from visibility rates.diff --git a/src/client/lib/error-messages.ts b/src/client/lib/error-messages.tsindex 940f8c1c..524afc3b 100644--- a/src/client/lib/error-messages.ts+++ b/src/client/lib/error-messages.ts@@ -1,5 +1,14 @@import { FREE_MAX_AUDIT_PAGES } from "@/shared/audit-limits";import { isErrorCode, type ErrorCode } from "@/shared/error-codes";+import { isHostedClientAuthMode } from "@/lib/auth-mode";++/**+ * Copy for an error we did not expect. Hosted users cannot read our logs, so+ * they get support; a self-hoster runs the server and can.+ */+export const UNEXPECTED_ERROR_MESSAGE = isHostedClientAuthMode()+ ? "Something went wrong on our end. Please try again, and contact support if it keeps happening."+ : "Something went wrong. Please try again, or check the server logs for details.";const STANDARD_MESSAGES: Record<ErrorCode, string> = {UNAUTHENTICATED: "Please sign in and try again.",@@ -34,8 +43,7 @@ const STANDARD_MESSAGES: Record<ErrorCode, string> = {UPSTREAM_UNAVAILABLE:"The data provider is temporarily unavailable. Please retry in a moment.",CONFLICT: "This request conflicts with existing data.",- INTERNAL_ERROR:- "An unexpected error occurred. Please check server logs and try again.",+ INTERNAL_ERROR: UNEXPECTED_ERROR_MESSAGE,};export function getStandardErrorMessage(diff --git a/src/routes/_app/index.tsx b/src/routes/_app/index.tsxindex 285fd48a..0559b900 100644--- a/src/routes/_app/index.tsx+++ b/src/routes/_app/index.tsx@@ -6,7 +6,10 @@ import {clearLastProjectId,getLastProjectId,} from "@/client/lib/active-project";-import { getErrorCode } from "@/client/lib/error-messages";+import {+ getErrorCode,+ UNEXPECTED_ERROR_MESSAGE,+} from "@/client/lib/error-messages";import { AuthErrorCard } from "@/client/components/AuthErrorCard";import { QueryError } from "@/client/components/QueryState";import { SkeletonPage } from "@/client/components/SkeletonPresets";@@ -72,7 +75,7 @@ function IndexRedirect() {<StatusScreen><QueryErrorerror={error}- fallback="An unexpected error occurred. Please check server logs."+ fallback={UNEXPECTED_ERROR_MESSAGE}onRetry={() => void refetch()}isRetrying={isFetching}/>diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.tsindex ad61d34f..dc556615 100644--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts@@ -62,6 +62,7 @@ const liveAnswer = {const providerFailure = {status: "failed",message: "Internal Server Error.",+ code: "UPSTREAM_UNAVAILABLE",retryable: true,} as const;@@ -206,7 +207,7 @@ describe("AI answer collection", () => {]);expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",- error: "Internal Server Error.",+ error: "UPSTREAM_UNAVAILABLE",});});@@ -282,7 +283,7 @@ describe("AI answer collection", () => {]);expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",- error: "Timed out",+ error: "INTERNAL_ERROR",});});@@ -314,11 +315,11 @@ describe("AI answer collection", () => {expect(dataforseo.llmResponse).toHaveBeenCalledTimes(4);expect(await repo.getObservation("answer-0")).toMatchObject({status: "failed",- error: "Internal Server Error.",+ error: "UPSTREAM_UNAVAILABLE",});expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",- error: "DataForSEO HTTP 502",+ error: "UPSTREAM_UNAVAILABLE",});});@@ -359,21 +360,18 @@ describe("AI answer collection", () => {});expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",- error: "Timed out",+ error: "INTERNAL_ERROR",});expect(dataforseo.trackingLiveBatch).toHaveBeenCalledOnce();});it("asks a live answer again when DataForSEO's backend fails it, up to 3 attempts", async () => {vi.useFakeTimers({ toFake: ["setTimeout"] });- const failed = (message: string) => ({- status: "fulfilled",- value: { ...providerFailure, message },- });+ const failed = { status: "fulfilled", value: providerFailure };dataforseo.trackingLiveBatch- .mockResolvedValueOnce([failed("Internal Server Error."), failed("1")])- .mockResolvedValueOnce([liveAnswer, failed("2")])- .mockResolvedValueOnce([failed("Internal SE Server Error.")]);+ .mockResolvedValueOnce([failed, failed])+ .mockResolvedValueOnce([liveAnswer, failed])+ .mockResolvedValueOnce([failed]);const collecting = collectAiLiveBatch(runId, customer, market, [{ tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },@@ -388,7 +386,7 @@ describe("AI answer collection", () => {});expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",- error: "Internal SE Server Error.",+ error: "UPSTREAM_UNAVAILABLE",});});diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.tsindex d28f4a1a..ecf7dbfb 100644--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts@@ -9,8 +9,9 @@ import {} from "@/server/lib/dataforseo";import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";import { resolveLatestLlmModelName } from "@/server/lib/dataforseo/llm-models";-import { AppError } from "@/server/lib/errors";+import { AppError, asAppError } from "@/server/lib/errors";import { isHostedServerAuthMode } from "@/server/lib/runtime-env";+import type { ErrorCode } from "@/shared/error-codes";import { getIsoCountryCode } from "@/shared/keyword-locations";import {aiEngineUsesModelApi,@@ -25,6 +26,7 @@ import {} from "../providers/dataforseoEvidence";import { aiBrands } from "./aiVisibilityConfiguration";import { aiCostForAnswers } from "./aiVisibilityCost";+import { AI_ANSWER_FAILURES } from "./aiVisibilityErrors";import { matchAiBrand } from "./aiVisibilityMatching";import { failAiRun } from "./aiVisibilityRuns";@@ -163,7 +165,8 @@ export async function postAiBatch(batch: AiTaskBatch,): Promise<AiPendingTask[]> {let posted: PostedAiTrackingTask[] = [];- let error = "DataForSEO did not accept this prompt.";+ // A rejected entry is logged where it is parsed.+ let error: ErrorCode = "INTERNAL_ERROR";try {posted = await createDataforseoClient(customer).aiSearch.trackingTaskPost({engine: batch.engine,@@ -171,7 +174,8 @@ export async function postAiBatch(...market,});} catch (cause) {- error = errorMessage(cause);+ error = asAppError(cause)?.code ?? "INTERNAL_ERROR";+ console.warn(`[ai-visibility] ${runId} task_post failed:`, cause);}const taskIds = new Map(posted.map((task) => [task.tag, task.taskId]));const rejected = batch.tasks@@ -251,6 +255,7 @@ async function fetchLiveAnswers(: {status: "failed",message: errorMessage(outcome.reason),+ code: asAppError(outcome.reason)?.code ?? "INTERNAL_ERROR",retryable: false,},);@@ -303,9 +308,12 @@ export async function collectAiModelBatch(await waitBeforeRetry();continue;}- await repo.failPendingObservations(runId, errorMessage(cause), [- task.tag,- ]);+ console.warn(+ `[ai-visibility] ${runId} answer ${task.tag} failed:`,+ cause,+ );+ const code = asAppError(cause)?.code ?? "INTERNAL_ERROR";+ await repo.failPendingObservations(runId, code, [task.tag]);return;}}@@ -381,16 +389,18 @@ async function saveAiOutcome(brands: AiBrand[],) {if (outcome.status === "failed") {- await repo.failPendingObservations(runId, outcome.message, [task.tag]);+ console.warn(+ `[ai-visibility] ${runId} answer ${task.tag} failed: ${outcome.message}`,+ );+ await repo.failPendingObservations(runId, outcome.code, [task.tag]);return;}const answer = parseDataforseoAnswer(outcome.result, task.engine);if (!answer) {- await repo.failPendingObservations(- runId,- "DataForSEO returned an answer we could not read.",- [task.tag],+ console.error(+ `[ai-visibility] ${runId} answer ${task.tag} has a shape we cannot read`,);+ await repo.failPendingObservations(runId, "INTERNAL_ERROR", [task.tag]);return;}await saveAiAnswer(task.tag, answer, brands);@@ -428,10 +438,7 @@ async function saveAiAnswer(export async function finalizeAiRun(runId: string) {const run = await repo.getRunInternal(runId);if (!run || run.status !== "running") return;- await repo.failPendingObservations(- runId,- "No answer arrived within the collection window.",- );+ await repo.failPendingObservations(runId, AI_ANSWER_FAILURES.notArrived);const statuses = await repo.getObservationStatuses([runId]);const completed = statuses.filter((row) => row.status === "completed").length;const status =@@ -452,8 +459,8 @@ export async function finalizeAiRun(runId: string) {export async function markAiRunFailed(runId: string, error: unknown) {const message =error instanceof AppError && error.code === "INSUFFICIENT_CREDITS"- ? "Not enough credits for the last check."- : "The last check could not collect answers.";+ ? AI_ANSWER_FAILURES.noCredits+ : AI_ANSWER_FAILURES.runFailed;await failAiRun(runId, message);const run = await repo.getRunInternal(runId);if (run) await repo.updateTracker(run.trackerId, { lastSkipReason: message });diff --git a/src/server/features/ai-visibility/services/aiVisibilityErrors.ts b/src/server/features/ai-visibility/services/aiVisibilityErrors.tsindex 22f79e74..11bdbc22 100644--- a/src/server/features/ai-visibility/services/aiVisibilityErrors.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityErrors.ts@@ -1,4 +1,5 @@import { AppError } from "@/server/lib/errors";+import { isErrorCode } from "@/shared/error-codes";export class AiVisibilityError extends AppError {constructor(@@ -9,3 +10,38 @@ export class AiVisibilityError extends AppError {super("AI_VISIBILITY_ERROR", message);}}++/**+ * Our copy for an answer that failed for a reason with no error code. A failed+ * answer stores this copy or an error code, never a provider's message.+ */+export const AI_ANSWER_FAILURES = {+ notArrived: "No answer arrived within the collection window.",+ runFailed: "The last check could not collect answers.",+ noCredits: "Not enough credits for the last check.",+ notStarted: "Collection could not start.",+ stoppedBeforeStart: "Collection stopped before it started.",+ stopped: "Collection stopped before it finished.",+} as const;++const FAILURE_COPY = new Set<string>(Object.values(AI_ANSWER_FAILURES));++/**+ * The error results show for an answer: an error code the client maps to its+ * own copy, or our failure copy. Older answers stored raw text. Our own+ * failures and the provider's rejections of a request we got wrong ("Invalid+ * 'location'.") read as internal errors. Any other text was a provider outage,+ * such as "Internal Server Error.".+ */+export function publicAiAnswerError(error: string | null): string | null {+ if (error === null || isErrorCode(error) || FAILURE_COPY.has(error))+ return error;+ if (error.startsWith("Collection stopped")) return AI_ANSWER_FAILURES.stopped;+ if (+ error === "DataForSEO did not accept this prompt." ||+ error === "DataForSEO returned an answer we could not read." ||+ error.startsWith("Invalid ")+ )+ return "INTERNAL_ERROR";+ return "UPSTREAM_UNAVAILABLE";+}diff --git a/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts b/src/server/features/ai-visibility/services/aiVisibilityResults.test.tsindex f057b818..40ef4862 100644--- a/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts@@ -28,7 +28,8 @@ const own = { name: "OpenSEO", domain: "openseo.so", own: true };const rival = { name: "Ahrefs", domain: "ahrefs.com", own: false };type Answer = {markdown?: string | null;- failed?: boolean;+ /** The stored error of a failed answer. */+ failed?: string;mentioned?: string[];cited?: string[];sources?: string[];@@ -78,7 +79,7 @@ async function seedRun(await repo.persistAnswer({observationId: id,values: answer.failed- ? { status: "failed", error: "Provider failed" }+ ? { status: "failed", error: answer.failed }: { status: "completed", answerMarkdown: answer.markdown ?? null },sources: (answer.sources ?? []).map((url, position) => ({id: crypto.randomUUID(),@@ -118,7 +119,7 @@ describe("AI visibility results", () => {{ markdown: "OpenSEO and Ahrefs", mentioned: ["OpenSEO", "Ahrefs"] },{ markdown: "Ahrefs", mentioned: ["Ahrefs"], cited: ["OpenSEO"] },{ markdown: null },- { failed: true },+ { failed: "UPSTREAM_UNAVAILABLE" },]);const results = await loadAiResults({@@ -148,6 +149,26 @@ describe("AI visibility results", () => {]);});+ it("never shows the raw text stored by an older answer", async () => {+ await seedRun("baseline", [+ { failed: "Internal SE Server Error." },+ { failed: "DataForSEO returned an answer we could not read." },+ { failed: "No answer arrived within the collection window." },+ ]);++ const results = await loadAiResults({+ ...query,+ includeHistory: false,+ competitorGap: false,+ });++ expect(results.rows.map((row) => row.error)).toEqual([+ "UPSTREAM_UNAVAILABLE",+ "INTERNAL_ERROR",+ "No answer arrived within the collection window.",+ ]);+ });+it("reads the latest scheduled run by default, not a newer manual check", async () => {const baseline = await seedRun("baseline", [{ markdown: "x" }]);await seedRun("manual", [{ markdown: "y" }], "2026-09-06T12:00:00.000Z");diff --git a/src/server/features/ai-visibility/services/aiVisibilityResults.ts b/src/server/features/ai-visibility/services/aiVisibilityResults.tsindex 3b2ca15f..256c8d51 100644--- a/src/server/features/ai-visibility/services/aiVisibilityResults.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityResults.ts@@ -23,7 +23,7 @@ import {} from "../repositories/AiVisibilityRepository";import type { ObservationWithPrompt } from "../repositories/AiVisibilityRepository";import { markdownToText } from "../providers/dataforseoEvidence";-import { AiVisibilityError } from "./aiVisibilityErrors";+import { AiVisibilityError, publicAiAnswerError } from "./aiVisibilityErrors";import {aiBrandNames,aiMentionSpans,@@ -75,7 +75,7 @@ function rowsFor(? "answered": "no_answer",collectedAt: o.collectedAt,- error: o.error,+ error: publicAiAnswerError(o.error),citationCount: sources[o.id]?.length ?? 0,brands: (matches[o.id] ?? []).map((m) => ({name: m.name,diff --git a/src/server/features/ai-visibility/services/aiVisibilityRuns.ts b/src/server/features/ai-visibility/services/aiVisibilityRuns.tsindex 9f7325c4..70bc7687 100644--- a/src/server/features/ai-visibility/services/aiVisibilityRuns.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityRuns.ts@@ -4,7 +4,7 @@ import {type ConfigurationRows,type RunRow,} from "../repositories/AiVisibilityRepository";-import { AiVisibilityError } from "./aiVisibilityErrors";+import { AI_ANSWER_FAILURES, AiVisibilityError } from "./aiVisibilityErrors";import { aiScope } from "./aiVisibilityConfiguration";import {getOptionalEnvValue,@@ -38,9 +38,9 @@ async function staleRunReason(run: RunRow): Promise<string | null> {.catch(() => null);if (status && ACTIVE_WORKFLOW_STATUSES.has(status.status)) return null;if (Date.now() - Date.parse(run.createdAt) < STARTUP_GRACE_MS) return null;- return status- ? `Collection stopped (${status.status}).`- : "Collection stopped before it started.";+ if (!status) return AI_ANSWER_FAILURES.stoppedBeforeStart;+ console.warn(`[ai-visibility] ${run.id} workflow stopped (${status.status})`);+ return AI_ANSWER_FAILURES.stopped;}/** Fails a run and its unfinished answers. Safe on a finished run. */@@ -143,7 +143,7 @@ export async function startRun(},});} catch (error) {- await failAiRun(id, "Collection could not start.");+ await failAiRun(id, AI_ANSWER_FAILURES.notStarted);throw error;}return getRun({ projectId: project.id, runId: id });diff --git a/src/server/lib/dataforseo/ai-tracking.ts b/src/server/lib/dataforseo/ai-tracking.tsindex eb68a98c..145351ee 100644--- a/src/server/lib/dataforseo/ai-tracking.ts+++ b/src/server/lib/dataforseo/ai-tracking.ts@@ -9,6 +9,7 @@ import {failedResponseError,} from "@/server/lib/dataforseo/envelope";import { AppError } from "@/server/lib/errors";+import type { ErrorCode } from "@/shared/error-codes";import {aiEngineUsesModelApi,type AiEngine,@@ -143,7 +144,9 @@ export async function postAiTrackingTasks(export type AiTrackingAnswer =| {status: "failed";+ /** The provider's message, for the logs. Users see `code`. */message: string;+ code: ErrorCode;/** DataForSEO's own backend failed the task, so a new one can succeed. */retryable: boolean;}@@ -175,11 +178,14 @@ function aiTrackingAnswer(): AiTrackingAnswer {if (task.status_code !== 20000) {if (isNoResultsTask(task)) return { status: "completed", result: null };+ const upstream = isUpstreamFailureStatus(task.status_code);return {status: "failed",message:task.status_message || `DataForSEO task failed (${task.status_code})`,- retryable: isUpstreamFailureStatus(task.status_code),+ // Like assertOk: the provider's own failure, or a request we got wrong.+ code: upstream ? "UPSTREAM_UNAVAILABLE" : "INTERNAL_ERROR",+ retryable: upstream,};}return { status: "completed", result: task.result?.[0] ?? null };