Skip to content

Open source alternative to Semrush and Ahrefs

Show standard copy instead of provider text when an AI visibility answer fails (#928)

Carter Smith2249317f767a
diff --git a/src/client/features/ai-visibility/AnswerContent.tsx b/src/client/features/ai-visibility/AnswerContent.tsx
index bc30cfd2..6c6aad43 100644
--- a/src/client/features/ai-visibility/AnswerContent.tsx
+++ b/src/client/features/ai-visibility/AnswerContent.tsx
@@ -11,6 +11,7 @@ import {
AccordionTrigger,
} from "@/client/components/ui/accordion";
import { Badge } from "@/client/components/ui/badge";
+import { getStandardErrorMessage } from "@/client/lib/error-messages";
import { Button } from "@/client/components/ui/button";
import {
Tabs,
@@ -77,7 +78,7 @@ function AnswerEvidence({ answer }: { answer: AiAnswer }) {
role="alert"
className="border-b bg-destructive/5 px-5 py-3 text-sm text-destructive border-border"
>
- {answer.observation.error}
+ {getStandardErrorMessage(new Error(answer.observation.error))}
</p>
)}
{answer.truncated && (
diff --git a/src/client/features/ai-visibility/PromptAnswers.tsx b/src/client/features/ai-visibility/PromptAnswers.tsx
index 8aa66021..26c40a4c 100644
--- a/src/client/features/ai-visibility/PromptAnswers.tsx
+++ b/src/client/features/ai-visibility/PromptAnswers.tsx
@@ -5,6 +5,7 @@ import {
type AiObservationRow,
type AiPrompt,
} from "@/shared/ai-visibility";
+import { getStandardErrorMessage } from "@/client/lib/error-messages";
import { AnswerContent } from "./AnswerContent";
export function PromptAnswers({
@@ -49,8 +50,9 @@ export function PromptAnswers({
{aiObservationStatusLabel(observation.status)}
</p>
<p className="text-sm text-muted-foreground">
- {observation.error ??
- "An answer is not available for this collection yet."}
+ {observation.error
+ ? getStandardErrorMessage(new Error(observation.error))
+ : "An answer is not available for this collection yet."}
</p>
<p className="text-xs text-muted-foreground">
This collection is excluded from visibility rates.
diff --git a/src/client/lib/error-messages.ts b/src/client/lib/error-messages.ts
index 940f8c1c..524afc3b 100644
--- a/src/client/lib/error-messages.ts
+++ b/src/client/lib/error-messages.ts
@@ -1,5 +1,14 @@
import { FREE_MAX_AUDIT_PAGES } from "@/shared/audit-limits";
import { isErrorCode, type ErrorCode } from "@/shared/error-codes";
+import { isHostedClientAuthMode } from "@/lib/auth-mode";
+
+/**
+ * Copy for an error we did not expect. Hosted users cannot read our logs, so
+ * they get support; a self-hoster runs the server and can.
+ */
+export const UNEXPECTED_ERROR_MESSAGE = isHostedClientAuthMode()
+ ? "Something went wrong on our end. Please try again, and contact support if it keeps happening."
+ : "Something went wrong. Please try again, or check the server logs for details.";
const STANDARD_MESSAGES: Record<ErrorCode, string> = {
UNAUTHENTICATED: "Please sign in and try again.",
@@ -34,8 +43,7 @@ const STANDARD_MESSAGES: Record<ErrorCode, string> = {
UPSTREAM_UNAVAILABLE:
"The data provider is temporarily unavailable. Please retry in a moment.",
CONFLICT: "This request conflicts with existing data.",
- INTERNAL_ERROR:
- "An unexpected error occurred. Please check server logs and try again.",
+ INTERNAL_ERROR: UNEXPECTED_ERROR_MESSAGE,
};
export function getStandardErrorMessage(
diff --git a/src/routes/_app/index.tsx b/src/routes/_app/index.tsx
index 285fd48a..0559b900 100644
--- a/src/routes/_app/index.tsx
+++ b/src/routes/_app/index.tsx
@@ -6,7 +6,10 @@ import {
clearLastProjectId,
getLastProjectId,
} from "@/client/lib/active-project";
-import { getErrorCode } from "@/client/lib/error-messages";
+import {
+ getErrorCode,
+ UNEXPECTED_ERROR_MESSAGE,
+} from "@/client/lib/error-messages";
import { AuthErrorCard } from "@/client/components/AuthErrorCard";
import { QueryError } from "@/client/components/QueryState";
import { SkeletonPage } from "@/client/components/SkeletonPresets";
@@ -72,7 +75,7 @@ function IndexRedirect() {
<StatusScreen>
<QueryError
error={error}
- fallback="An unexpected error occurred. Please check server logs."
+ fallback={UNEXPECTED_ERROR_MESSAGE}
onRetry={() => void refetch()}
isRetrying={isFetching}
/>
diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
index ad61d34f..dc556615 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
@@ -62,6 +62,7 @@ const liveAnswer = {
const providerFailure = {
status: "failed",
message: "Internal Server Error.",
+ code: "UPSTREAM_UNAVAILABLE",
retryable: true,
} as const;
@@ -206,7 +207,7 @@ describe("AI answer collection", () => {
]);
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
- error: "Internal Server Error.",
+ error: "UPSTREAM_UNAVAILABLE",
});
});
@@ -282,7 +283,7 @@ describe("AI answer collection", () => {
]);
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
- error: "Timed out",
+ error: "INTERNAL_ERROR",
});
});
@@ -314,11 +315,11 @@ describe("AI answer collection", () => {
expect(dataforseo.llmResponse).toHaveBeenCalledTimes(4);
expect(await repo.getObservation("answer-0")).toMatchObject({
status: "failed",
- error: "Internal Server Error.",
+ error: "UPSTREAM_UNAVAILABLE",
});
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
- error: "DataForSEO HTTP 502",
+ error: "UPSTREAM_UNAVAILABLE",
});
});
@@ -359,21 +360,18 @@ describe("AI answer collection", () => {
});
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
- error: "Timed out",
+ error: "INTERNAL_ERROR",
});
expect(dataforseo.trackingLiveBatch).toHaveBeenCalledOnce();
});
it("asks a live answer again when DataForSEO's backend fails it, up to 3 attempts", async () => {
vi.useFakeTimers({ toFake: ["setTimeout"] });
- const failed = (message: string) => ({
- status: "fulfilled",
- value: { ...providerFailure, message },
- });
+ const failed = { status: "fulfilled", value: providerFailure };
dataforseo.trackingLiveBatch
- .mockResolvedValueOnce([failed("Internal Server Error."), failed("1")])
- .mockResolvedValueOnce([liveAnswer, failed("2")])
- .mockResolvedValueOnce([failed("Internal SE Server Error.")]);
+ .mockResolvedValueOnce([failed, failed])
+ .mockResolvedValueOnce([liveAnswer, failed])
+ .mockResolvedValueOnce([failed]);
const collecting = collectAiLiveBatch(runId, customer, market, [
{ tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },
@@ -388,7 +386,7 @@ describe("AI answer collection", () => {
});
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
- error: "Internal SE Server Error.",
+ error: "UPSTREAM_UNAVAILABLE",
});
});
diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
index d28f4a1a..ecf7dbfb 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
@@ -9,8 +9,9 @@ import {
} from "@/server/lib/dataforseo";
import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
import { resolveLatestLlmModelName } from "@/server/lib/dataforseo/llm-models";
-import { AppError } from "@/server/lib/errors";
+import { AppError, asAppError } from "@/server/lib/errors";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
+import type { ErrorCode } from "@/shared/error-codes";
import { getIsoCountryCode } from "@/shared/keyword-locations";
import {
aiEngineUsesModelApi,
@@ -25,6 +26,7 @@ import {
} from "../providers/dataforseoEvidence";
import { aiBrands } from "./aiVisibilityConfiguration";
import { aiCostForAnswers } from "./aiVisibilityCost";
+import { AI_ANSWER_FAILURES } from "./aiVisibilityErrors";
import { matchAiBrand } from "./aiVisibilityMatching";
import { failAiRun } from "./aiVisibilityRuns";
@@ -163,7 +165,8 @@ export async function postAiBatch(
batch: AiTaskBatch,
): Promise<AiPendingTask[]> {
let posted: PostedAiTrackingTask[] = [];
- let error = "DataForSEO did not accept this prompt.";
+ // A rejected entry is logged where it is parsed.
+ let error: ErrorCode = "INTERNAL_ERROR";
try {
posted = await createDataforseoClient(customer).aiSearch.trackingTaskPost({
engine: batch.engine,
@@ -171,7 +174,8 @@ export async function postAiBatch(
...market,
});
} catch (cause) {
- error = errorMessage(cause);
+ error = asAppError(cause)?.code ?? "INTERNAL_ERROR";
+ console.warn(`[ai-visibility] ${runId} task_post failed:`, cause);
}
const taskIds = new Map(posted.map((task) => [task.tag, task.taskId]));
const rejected = batch.tasks
@@ -251,6 +255,7 @@ async function fetchLiveAnswers(
: {
status: "failed",
message: errorMessage(outcome.reason),
+ code: asAppError(outcome.reason)?.code ?? "INTERNAL_ERROR",
retryable: false,
},
);
@@ -303,9 +308,12 @@ export async function collectAiModelBatch(
await waitBeforeRetry();
continue;
}
- await repo.failPendingObservations(runId, errorMessage(cause), [
- task.tag,
- ]);
+ console.warn(
+ `[ai-visibility] ${runId} answer ${task.tag} failed:`,
+ cause,
+ );
+ const code = asAppError(cause)?.code ?? "INTERNAL_ERROR";
+ await repo.failPendingObservations(runId, code, [task.tag]);
return;
}
}
@@ -381,16 +389,18 @@ async function saveAiOutcome(
brands: AiBrand[],
) {
if (outcome.status === "failed") {
- await repo.failPendingObservations(runId, outcome.message, [task.tag]);
+ console.warn(
+ `[ai-visibility] ${runId} answer ${task.tag} failed: ${outcome.message}`,
+ );
+ await repo.failPendingObservations(runId, outcome.code, [task.tag]);
return;
}
const answer = parseDataforseoAnswer(outcome.result, task.engine);
if (!answer) {
- await repo.failPendingObservations(
- runId,
- "DataForSEO returned an answer we could not read.",
- [task.tag],
+ console.error(
+ `[ai-visibility] ${runId} answer ${task.tag} has a shape we cannot read`,
);
+ await repo.failPendingObservations(runId, "INTERNAL_ERROR", [task.tag]);
return;
}
await saveAiAnswer(task.tag, answer, brands);
@@ -428,10 +438,7 @@ async function saveAiAnswer(
export async function finalizeAiRun(runId: string) {
const run = await repo.getRunInternal(runId);
if (!run || run.status !== "running") return;
- await repo.failPendingObservations(
- runId,
- "No answer arrived within the collection window.",
- );
+ await repo.failPendingObservations(runId, AI_ANSWER_FAILURES.notArrived);
const statuses = await repo.getObservationStatuses([runId]);
const completed = statuses.filter((row) => row.status === "completed").length;
const status =
@@ -452,8 +459,8 @@ export async function finalizeAiRun(runId: string) {
export async function markAiRunFailed(runId: string, error: unknown) {
const message =
error instanceof AppError && error.code === "INSUFFICIENT_CREDITS"
- ? "Not enough credits for the last check."
- : "The last check could not collect answers.";
+ ? AI_ANSWER_FAILURES.noCredits
+ : AI_ANSWER_FAILURES.runFailed;
await failAiRun(runId, message);
const run = await repo.getRunInternal(runId);
if (run) await repo.updateTracker(run.trackerId, { lastSkipReason: message });
diff --git a/src/server/features/ai-visibility/services/aiVisibilityErrors.ts b/src/server/features/ai-visibility/services/aiVisibilityErrors.ts
index 22f79e74..11bdbc22 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityErrors.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityErrors.ts
@@ -1,4 +1,5 @@
import { AppError } from "@/server/lib/errors";
+import { isErrorCode } from "@/shared/error-codes";
export class AiVisibilityError extends AppError {
constructor(
@@ -9,3 +10,38 @@ export class AiVisibilityError extends AppError {
super("AI_VISIBILITY_ERROR", message);
}
}
+
+/**
+ * Our copy for an answer that failed for a reason with no error code. A failed
+ * answer stores this copy or an error code, never a provider's message.
+ */
+export const AI_ANSWER_FAILURES = {
+ notArrived: "No answer arrived within the collection window.",
+ runFailed: "The last check could not collect answers.",
+ noCredits: "Not enough credits for the last check.",
+ notStarted: "Collection could not start.",
+ stoppedBeforeStart: "Collection stopped before it started.",
+ stopped: "Collection stopped before it finished.",
+} as const;
+
+const FAILURE_COPY = new Set<string>(Object.values(AI_ANSWER_FAILURES));
+
+/**
+ * The error results show for an answer: an error code the client maps to its
+ * own copy, or our failure copy. Older answers stored raw text. Our own
+ * failures and the provider's rejections of a request we got wrong ("Invalid
+ * 'location'.") read as internal errors. Any other text was a provider outage,
+ * such as "Internal Server Error.".
+ */
+export function publicAiAnswerError(error: string | null): string | null {
+ if (error === null || isErrorCode(error) || FAILURE_COPY.has(error))
+ return error;
+ if (error.startsWith("Collection stopped")) return AI_ANSWER_FAILURES.stopped;
+ if (
+ error === "DataForSEO did not accept this prompt." ||
+ error === "DataForSEO returned an answer we could not read." ||
+ error.startsWith("Invalid ")
+ )
+ return "INTERNAL_ERROR";
+ return "UPSTREAM_UNAVAILABLE";
+}
diff --git a/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts b/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts
index f057b818..40ef4862 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityResults.test.ts
@@ -28,7 +28,8 @@ const own = { name: "OpenSEO", domain: "openseo.so", own: true };
const rival = { name: "Ahrefs", domain: "ahrefs.com", own: false };
type Answer = {
markdown?: string | null;
- failed?: boolean;
+ /** The stored error of a failed answer. */
+ failed?: string;
mentioned?: string[];
cited?: string[];
sources?: string[];
@@ -78,7 +79,7 @@ async function seedRun(
await repo.persistAnswer({
observationId: id,
values: answer.failed
- ? { status: "failed", error: "Provider failed" }
+ ? { status: "failed", error: answer.failed }
: { status: "completed", answerMarkdown: answer.markdown ?? null },
sources: (answer.sources ?? []).map((url, position) => ({
id: crypto.randomUUID(),
@@ -118,7 +119,7 @@ describe("AI visibility results", () => {
{ markdown: "OpenSEO and Ahrefs", mentioned: ["OpenSEO", "Ahrefs"] },
{ markdown: "Ahrefs", mentioned: ["Ahrefs"], cited: ["OpenSEO"] },
{ markdown: null },
- { failed: true },
+ { failed: "UPSTREAM_UNAVAILABLE" },
]);
const results = await loadAiResults({
@@ -148,6 +149,26 @@ describe("AI visibility results", () => {
]);
});
+ it("never shows the raw text stored by an older answer", async () => {
+ await seedRun("baseline", [
+ { failed: "Internal SE Server Error." },
+ { failed: "DataForSEO returned an answer we could not read." },
+ { failed: "No answer arrived within the collection window." },
+ ]);
+
+ const results = await loadAiResults({
+ ...query,
+ includeHistory: false,
+ competitorGap: false,
+ });
+
+ expect(results.rows.map((row) => row.error)).toEqual([
+ "UPSTREAM_UNAVAILABLE",
+ "INTERNAL_ERROR",
+ "No answer arrived within the collection window.",
+ ]);
+ });
+
it("reads the latest scheduled run by default, not a newer manual check", async () => {
const baseline = await seedRun("baseline", [{ markdown: "x" }]);
await seedRun("manual", [{ markdown: "y" }], "2026-09-06T12:00:00.000Z");
diff --git a/src/server/features/ai-visibility/services/aiVisibilityResults.ts b/src/server/features/ai-visibility/services/aiVisibilityResults.ts
index 3b2ca15f..256c8d51 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityResults.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityResults.ts
@@ -23,7 +23,7 @@ import {
} from "../repositories/AiVisibilityRepository";
import type { ObservationWithPrompt } from "../repositories/AiVisibilityRepository";
import { markdownToText } from "../providers/dataforseoEvidence";
-import { AiVisibilityError } from "./aiVisibilityErrors";
+import { AiVisibilityError, publicAiAnswerError } from "./aiVisibilityErrors";
import {
aiBrandNames,
aiMentionSpans,
@@ -75,7 +75,7 @@ function rowsFor(
? "answered"
: "no_answer",
collectedAt: o.collectedAt,
- error: o.error,
+ error: publicAiAnswerError(o.error),
citationCount: sources[o.id]?.length ?? 0,
brands: (matches[o.id] ?? []).map((m) => ({
name: m.name,
diff --git a/src/server/features/ai-visibility/services/aiVisibilityRuns.ts b/src/server/features/ai-visibility/services/aiVisibilityRuns.ts
index 9f7325c4..70bc7687 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityRuns.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityRuns.ts
@@ -4,7 +4,7 @@ import {
type ConfigurationRows,
type RunRow,
} from "../repositories/AiVisibilityRepository";
-import { AiVisibilityError } from "./aiVisibilityErrors";
+import { AI_ANSWER_FAILURES, AiVisibilityError } from "./aiVisibilityErrors";
import { aiScope } from "./aiVisibilityConfiguration";
import {
getOptionalEnvValue,
@@ -38,9 +38,9 @@ async function staleRunReason(run: RunRow): Promise<string | null> {
.catch(() => null);
if (status && ACTIVE_WORKFLOW_STATUSES.has(status.status)) return null;
if (Date.now() - Date.parse(run.createdAt) < STARTUP_GRACE_MS) return null;
- return status
- ? `Collection stopped (${status.status}).`
- : "Collection stopped before it started.";
+ if (!status) return AI_ANSWER_FAILURES.stoppedBeforeStart;
+ console.warn(`[ai-visibility] ${run.id} workflow stopped (${status.status})`);
+ return AI_ANSWER_FAILURES.stopped;
}
/** Fails a run and its unfinished answers. Safe on a finished run. */
@@ -143,7 +143,7 @@ export async function startRun(
},
});
} catch (error) {
- await failAiRun(id, "Collection could not start.");
+ await failAiRun(id, AI_ANSWER_FAILURES.notStarted);
throw error;
}
return getRun({ projectId: project.id, runId: id });
diff --git a/src/server/lib/dataforseo/ai-tracking.ts b/src/server/lib/dataforseo/ai-tracking.ts
index eb68a98c..145351ee 100644
--- a/src/server/lib/dataforseo/ai-tracking.ts
+++ b/src/server/lib/dataforseo/ai-tracking.ts
@@ -9,6 +9,7 @@ import {
failedResponseError,
} from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors";
+import type { ErrorCode } from "@/shared/error-codes";
import {
aiEngineUsesModelApi,
type AiEngine,
@@ -143,7 +144,9 @@ export async function postAiTrackingTasks(
export type AiTrackingAnswer =
| {
status: "failed";
+ /** The provider's message, for the logs. Users see `code`. */
message: string;
+ code: ErrorCode;
/** DataForSEO's own backend failed the task, so a new one can succeed. */
retryable: boolean;
}
@@ -175,11 +178,14 @@ function aiTrackingAnswer(
): AiTrackingAnswer {
if (task.status_code !== 20000) {
if (isNoResultsTask(task)) return { status: "completed", result: null };
+ const upstream = isUpstreamFailureStatus(task.status_code);
return {
status: "failed",
message:
task.status_message || `DataForSEO task failed (${task.status_code})`,
- retryable: isUpstreamFailureStatus(task.status_code),
+ // Like assertOk: the provider's own failure, or a request we got wrong.
+ code: upstream ? "UPSTREAM_UNAVAILABLE" : "INTERNAL_ERROR",
+ retryable: upstream,
};
}
return { status: "completed", result: task.result?.[0] ?? null };