Skip to content

Open source alternative to Semrush and Ahrefs

Retry AI visibility answers that DataForSEO's backend fails, up to 3 attempts (EVE-419) (#926)

Carter Smith95a5f3f2cdb4
diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
index 7971d282..ad61d34f 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts
@@ -1,5 +1,7 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { aiObservations, projectCompetitors } from "@/db/schema";
+import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
+import { AppError } from "@/server/lib/errors";
import { AiVisibilityRepository as repo } from "../repositories/AiVisibilityRepository";
import {
collectAiLiveBatch,
@@ -46,6 +48,26 @@ vi.mock("@/server/lib/dataforseo/llm-models", () => ({
const runId = "run";
const market = { locationCode: 2840, languageCode: "en" };
+const pendingTask = {
+ tag: "answer-0",
+ taskId: "task",
+ engine: "chatgpt",
+ prompt: "Which SEO tools?",
+ attempts: 1,
+} as const;
+const liveAnswer = {
+ status: "fulfilled",
+ value: { status: "completed", result: { markdown: "OpenSEO", sources: [] } },
+} as const;
+const providerFailure = {
+ status: "failed",
+ message: "Internal Server Error.",
+ retryable: true,
+} as const;
+
+afterEach(() => {
+ vi.useRealTimers();
+});
beforeEach(async () => {
await testDb.seedProject();
@@ -97,11 +119,9 @@ describe("AI answer collection", () => {
},
});
- const pending = await collectAiRound(runId, [
- { tag: "answer-0", taskId: "task", engine: "chatgpt" },
- ]);
+ const round = await collectAiRound(runId, [pendingTask], true);
- expect(pending).toEqual([]);
+ expect(round).toEqual({ pending: [], retry: [] });
expect(await repo.getObservation("answer-0")).toMatchObject({
status: "completed",
});
@@ -142,11 +162,51 @@ describe("AI answer collection", () => {
],
});
- expect(pending).toEqual([
- { tag: "answer-0", taskId: "task", engine: "chatgpt" },
+ expect(pending).toEqual([pendingTask]);
+ expect(await repo.getObservation("answer-1")).toMatchObject({
+ status: "failed",
+ });
+ });
+
+ it("counts the attempt of an answer posted again, so its retries stop at 3", async () => {
+ dataforseo.trackingTaskPost.mockResolvedValue([
+ { tag: "answer-0", taskId: "task" },
+ ]);
+
+ const pending = await postAiBatch(runId, customer, market, {
+ engine: "chatgpt",
+ tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 2 }],
+ });
+
+ expect(pending).toEqual([{ ...pendingTask, attempts: 3 }]);
+ });
+
+ it("returns a queued answer DataForSEO's backend failed for a new post, and fails it after the last attempt", async () => {
+ dataforseo.fetchTaskResult.mockResolvedValue(providerFailure);
+
+ const round = await collectAiRound(
+ runId,
+ [
+ pendingTask,
+ {
+ ...pendingTask,
+ tag: "answer-1",
+ prompt: "Best rank tracker?",
+ attempts: 3,
+ },
+ ],
+ true,
+ );
+
+ expect(round.retry).toEqual([
+ {
+ engine: "chatgpt",
+ tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 1 }],
+ },
]);
expect(await repo.getObservation("answer-1")).toMatchObject({
status: "failed",
+ error: "Internal Server Error.",
});
});
@@ -226,6 +286,42 @@ describe("AI answer collection", () => {
});
});
+ it("asks a model answer again only when DataForSEO returned the failed task", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout"] });
+ dataforseo.llmResponse.mockImplementation(
+ async ({ userPrompt }: { userPrompt: string }) => {
+ if (userPrompt === "Best rank tracker?")
+ throw new AppError("UPSTREAM_UNAVAILABLE", "DataForSEO HTTP 502");
+ throw new DataforseoChargedTaskError(
+ "Internal Server Error.",
+ { path: [], costUsd: 0 },
+ false,
+ "UPSTREAM_UNAVAILABLE",
+ );
+ },
+ );
+
+ const collecting = collectAiModelBatch(runId, customer, market, {
+ engine: "claude",
+ tasks: [
+ { tag: "answer-0", prompt: "Which SEO tools?" },
+ { tag: "answer-1", prompt: "Best rank tracker?" },
+ ],
+ });
+ await vi.runAllTimersAsync();
+ await collecting;
+
+ expect(dataforseo.llmResponse).toHaveBeenCalledTimes(4);
+ expect(await repo.getObservation("answer-0")).toMatchObject({
+ status: "failed",
+ error: "Internal Server Error.",
+ });
+ expect(await repo.getObservation("answer-1")).toMatchObject({
+ status: "failed",
+ error: "DataForSEO HTTP 502",
+ });
+ });
+
it("sends Claude to the model API, not the live scraper, on a manual run", async () => {
const [{ promptId }] = await repo.getObservations([runId]);
await testDb.db.insert(aiObservations).values({
@@ -249,13 +345,7 @@ describe("AI answer collection", () => {
it("collects a manual run live, saving answers and failing rejected calls", async () => {
expect(await prepareAiRun(runId, customer)).toMatchObject({ live: true });
dataforseo.trackingLiveBatch.mockResolvedValue([
- {
- status: "fulfilled",
- value: {
- status: "completed",
- result: { markdown: "OpenSEO", sources: [] },
- },
- },
+ liveAnswer,
{ status: "rejected", reason: new Error("Timed out") },
]);
@@ -271,17 +361,39 @@ describe("AI answer collection", () => {
status: "failed",
error: "Timed out",
});
+ expect(dataforseo.trackingLiveBatch).toHaveBeenCalledOnce();
});
- it("saves the rest of a paid live batch when one answer cannot be saved", async () => {
- const answer = {
+ it("asks a live answer again when DataForSEO's backend fails it, up to 3 attempts", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout"] });
+ const failed = (message: string) => ({
status: "fulfilled",
- value: {
- status: "completed",
- result: { markdown: "OpenSEO", sources: [] },
- },
- };
- dataforseo.trackingLiveBatch.mockResolvedValue([answer, answer]);
+ value: { ...providerFailure, message },
+ });
+ dataforseo.trackingLiveBatch
+ .mockResolvedValueOnce([failed("Internal Server Error."), failed("1")])
+ .mockResolvedValueOnce([liveAnswer, failed("2")])
+ .mockResolvedValueOnce([failed("Internal SE Server Error.")]);
+
+ const collecting = collectAiLiveBatch(runId, customer, market, [
+ { tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },
+ { tag: "answer-1", prompt: "Best rank tracker?", engine: "chatgpt" },
+ ]);
+ await vi.runAllTimersAsync();
+ await collecting;
+
+ expect(dataforseo.trackingLiveBatch).toHaveBeenCalledTimes(3);
+ expect(await repo.getObservation("answer-0")).toMatchObject({
+ status: "completed",
+ });
+ expect(await repo.getObservation("answer-1")).toMatchObject({
+ status: "failed",
+ error: "Internal SE Server Error.",
+ });
+ });
+
+ it("saves the rest of a paid live batch when one answer cannot be saved", async () => {
+ dataforseo.trackingLiveBatch.mockResolvedValue([liveAnswer, liveAnswer]);
vi.spyOn(repo, "persistAnswer").mockRejectedValueOnce(new Error("D1"));
vi.spyOn(console, "error").mockImplementation(() => {});
@@ -303,9 +415,7 @@ describe("AI answer collection", () => {
status: "completed",
result: { markdown: "OpenSEO", sources: [] },
});
- await collectAiRound(runId, [
- { tag: "answer-0", taskId: "task", engine: "chatgpt" },
- ]);
+ await collectAiRound(runId, [pendingTask], true);
await finalizeAiRun(runId);
diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
index 50426e0d..d28f4a1a 100644
--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts
@@ -7,6 +7,7 @@ import {
type AiTrackingAnswer,
type PostedAiTrackingTask,
} from "@/server/lib/dataforseo";
+import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
import { resolveLatestLlmModelName } from "@/server/lib/dataforseo/llm-models";
import { AppError } from "@/server/lib/errors";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
@@ -30,18 +31,22 @@ import { failAiRun } from "./aiVisibilityRuns";
// Each function here is one workflow step body: inputs are its parameters and
// the return value is what the workflow engine persists and replays.
-interface AiTaskBatch {
- engine: AiEngine;
- /** Tag = answer row id. At most 100 per task_post, 6 per model batch. */
- tasks: { tag: string; prompt: string }[];
-}
-/** One live answer. A live step mixes engines, so each task names its own. */
-interface AiLiveTask {
+interface AiTask {
+ /** The answer row id. */
tag: string;
prompt: string;
+ /** Attempts already made, when a queued answer is posted again. */
+ attempts?: number;
+}
+interface AiTaskBatch {
engine: AiEngine;
+ /** At most 100 per task_post, 6 per model batch. */
+ tasks: AiTask[];
}
-export type AiPendingTask = PostedAiTrackingTask & { engine: AiEngine };
+/** One live answer. A live step mixes engines, so each task names its own. */
+type AiLiveTask = AiTask & { engine: AiEngine };
+export type AiPendingTask = PostedAiTrackingTask &
+ AiLiveTask & { attempts: number };
interface AiRunMarket {
locationCode: number;
languageCode: string;
@@ -53,6 +58,22 @@ interface AiRunMarket {
*/
const LIVE_ANSWERS_PER_STEP = 6;
+/**
+ * Attempts per answer. A new attempt follows only when DataForSEO returned the
+ * task and its own backend failed it, so the failure and its cost are known. A
+ * transport error or a timeout can hide a charge, so it is never retried.
+ */
+const AI_ANSWER_ATTEMPTS = 3;
+/** Pause before a new attempt, so a short provider outage can clear. */
+const AI_RETRY_DELAY_MS = 3_000;
+
+const waitBeforeRetry = () =>
+ new Promise((resolve) => setTimeout(resolve, AI_RETRY_DELAY_MS));
+
+function errorMessage(error: unknown) {
+ return error instanceof Error ? error.message : String(error);
+}
+
/**
* Marks the run running and groups its answers into batches. Manual and
* baseline runs collect live answers so the user sees results sooner.
@@ -94,20 +115,20 @@ export async function prepareAiRun(
locationCode: run.locationCode,
languageCode: run.languageCode,
};
+ const tasks = pending.map((row) => ({
+ tag: row.id,
+ prompt: row.prompt,
+ engine: row.engine,
+ }));
const modelBatches = engineBatches(
- pending.filter((row) => aiEngineUsesModelApi(row.engine)),
+ tasks.filter((task) => aiEngineUsesModelApi(task.engine)),
LIVE_ANSWERS_PER_STEP,
);
- const answers = pending.filter((row) => !aiEngineUsesModelApi(row.engine));
+ const answers = tasks.filter((task) => !aiEngineUsesModelApi(task.engine));
if (live) {
- const tasks = answers.map((row) => ({
- tag: row.id,
- prompt: row.prompt,
- engine: row.engine,
- }));
const batches: AiLiveTask[][] = [];
- for (let i = 0; i < tasks.length; i += LIVE_ANSWERS_PER_STEP)
- batches.push(tasks.slice(i, i + LIVE_ANSWERS_PER_STEP));
+ for (let i = 0; i < answers.length; i += LIVE_ANSWERS_PER_STEP)
+ batches.push(answers.slice(i, i + LIVE_ANSWERS_PER_STEP));
return { market, modelBatches, live, batches };
}
return {
@@ -119,15 +140,12 @@ export async function prepareAiRun(
}
/** Groups answers by engine into batches of at most `size`. */
-function engineBatches(
- answers: { id: string; prompt: string; engine: AiEngine }[],
- size: number,
-): AiTaskBatch[] {
+function engineBatches(answers: AiLiveTask[], size: number): AiTaskBatch[] {
const batches: AiTaskBatch[] = [];
- for (const engine of new Set(answers.map((row) => row.engine))) {
+ for (const engine of new Set(answers.map((task) => task.engine))) {
const tasks = answers
- .filter((row) => row.engine === engine)
- .map((row) => ({ tag: row.id, prompt: row.prompt }));
+ .filter((task) => task.engine === engine)
+ .map(({ tag, prompt, attempts }) => ({ tag, prompt, attempts }));
for (let i = 0; i < tasks.length; i += size)
batches.push({ engine, tasks: tasks.slice(i, i + size) });
}
@@ -153,20 +171,26 @@ export async function postAiBatch(
...market,
});
} catch (cause) {
- error = cause instanceof Error ? cause.message : String(cause);
+ error = errorMessage(cause);
}
- const accepted = new Set(posted.map((task) => task.tag));
+ const taskIds = new Map(posted.map((task) => [task.tag, task.taskId]));
const rejected = batch.tasks
.map((task) => task.tag)
- .filter((tag) => !accepted.has(tag));
+ .filter((tag) => !taskIds.has(tag));
if (rejected.length)
await repo.failPendingObservations(runId, error, rejected);
- return posted.map((task) => ({ ...task, engine: batch.engine }));
+ return batch.tasks.flatMap((task) => {
+ const taskId = taskIds.get(task.tag);
+ if (!taskId) return [];
+ const attempts = (task.attempts ?? 0) + 1;
+ return [{ ...task, taskId, engine: batch.engine, attempts }];
+ });
}
/**
* Collects one batch of live answers through the metered client, which bills
- * them, and saves each one. Answers that fail now are not retried.
+ * them, and saves each one. An answer that DataForSEO's backend failed is
+ * asked again, up to AI_ANSWER_ATTEMPTS.
*/
export async function collectAiLiveBatch(
runId: string,
@@ -176,6 +200,43 @@ export async function collectAiLiveBatch(
) {
const brands = await runBrands(runId);
if (!brands) return;
+ let remaining = tasks;
+ for (let attempt = 1; remaining.length > 0; attempt++) {
+ if (attempt > 1) await waitBeforeRetry();
+ const answers = await fetchLiveAnswers(customer, market, remaining);
+ const retry: AiLiveTask[] = [];
+ for (const [index, result] of answers.entries()) {
+ const task = remaining[index];
+ if (
+ result.status === "failed" &&
+ result.retryable &&
+ attempt < AI_ANSWER_ATTEMPTS
+ ) {
+ retry.push(task);
+ continue;
+ }
+ // Every answer here is already paid for, and the step cannot rerun, so
+ // one failed write must not drop the rest. finalizeAiRun fails the
+ // answer that stays pending.
+ try {
+ await saveAiOutcome(runId, task, result, brands);
+ } catch (error) {
+ console.error(
+ `[ai-visibility] ${runId} could not save answer ${task.tag}:`,
+ error,
+ );
+ }
+ }
+ remaining = retry;
+ }
+}
+
+/** A call that throws becomes a failed answer that is not retried. */
+async function fetchLiveAnswers(
+ customer: BillingCustomerContext,
+ market: AiRunMarket,
+ tasks: AiLiveTask[],
+): Promise<AiTrackingAnswer[]> {
let settled: PromiseSettledResult<AiTrackingAnswer>[];
try {
settled = await createDataforseoClient(customer).aiSearch.trackingLiveBatch(
@@ -184,37 +245,24 @@ export async function collectAiLiveBatch(
} catch (error) {
settled = tasks.map(() => ({ status: "rejected", reason: error }));
}
- for (const [index, outcome] of settled.entries()) {
- const result: AiTrackingAnswer =
- outcome.status === "fulfilled"
- ? outcome.value
- : {
- status: "failed",
- message:
- outcome.reason instanceof Error
- ? outcome.reason.message
- : String(outcome.reason),
- };
- // Every answer here is already paid for, and the step cannot rerun, so
- // one failed write must not drop the rest. finalizeAiRun fails the
- // answer that stays pending.
- try {
- await saveAiOutcome(runId, tasks[index], result, brands);
- } catch (error) {
- console.error(
- `[ai-visibility] ${runId} could not save answer ${tasks[index].tag}:`,
- error,
- );
- }
- }
+ return settled.map((outcome) =>
+ outcome.status === "fulfilled"
+ ? outcome.value
+ : {
+ status: "failed",
+ message: errorMessage(outcome.reason),
+ retryable: false,
+ },
+ );
}
/**
* Asks each prompt of a Claude or Perplexity batch through the metered model
* API client, which bills each answer, and saves each answer as soon as it
* arrives, so a slow call that hits the step timeout cannot discard answers
- * already paid for. Model answers have no language setting: only the country
- * guides the web search.
+ * already paid for. An answer that DataForSEO's backend failed is asked again,
+ * up to AI_ANSWER_ATTEMPTS. Model answers have no language setting: only the
+ * country guides the web search.
*/
export async function collectAiModelBatch(
runId: string,
@@ -232,23 +280,34 @@ export async function collectAiModelBatch(
const country = getIsoCountryCode(market.locationCode).toUpperCase();
await Promise.all(
batch.tasks.map(async (task) => {
- try {
- const result = await client.aiSearch.llmResponse({
- modelSlug,
- modelName,
- userPrompt: task.prompt,
- webSearch: true,
- // Saving the tracker rejects countries the engine cannot search.
- webSearchCountryCode: country,
- maxOutputTokens: MODEL_MAX_OUTPUT_TOKENS,
- });
- await saveAiAnswer(task.tag, parseLlmResponseAnswer(result), brands);
- } catch (cause) {
- await repo.failPendingObservations(
- runId,
- cause instanceof Error ? cause.message : String(cause),
- [task.tag],
- );
+ for (let attempt = 1; ; attempt++) {
+ try {
+ const result = await client.aiSearch.llmResponse({
+ modelSlug,
+ modelName,
+ userPrompt: task.prompt,
+ webSearch: true,
+ // Saving the tracker rejects countries the engine cannot search.
+ webSearchCountryCode: country,
+ maxOutputTokens: MODEL_MAX_OUTPUT_TOKENS,
+ });
+ await saveAiAnswer(task.tag, parseLlmResponseAnswer(result), brands);
+ return;
+ } catch (cause) {
+ // A charged task error is a task DataForSEO returned with its cost.
+ if (
+ cause instanceof DataforseoChargedTaskError &&
+ cause.code === "UPSTREAM_UNAVAILABLE" &&
+ attempt < AI_ANSWER_ATTEMPTS
+ ) {
+ await waitBeforeRetry();
+ continue;
+ }
+ await repo.failPendingObservations(runId, errorMessage(cause), [
+ task.tag,
+ ]);
+ return;
+ }
}
}),
);
@@ -265,15 +324,19 @@ const TASK_GET_CONCURRENCY = 25;
/**
* Reads each pending answer once (task_get is free) and saves finished ones
- * with their citations and brand results. Returns the tasks still pending.
+ * with their citations and brand results. Returns the tasks still pending, and
+ * the answers DataForSEO's backend failed that `canRetry` lets the caller post
+ * again.
*/
export async function collectAiRound(
runId: string,
tasks: AiPendingTask[],
-): Promise<AiPendingTask[]> {
+ canRetry: boolean,
+): Promise<{ pending: AiPendingTask[]; retry: AiTaskBatch[] }> {
const brands = await runBrands(runId);
- if (!brands) return [];
- const stillPending: AiPendingTask[] = [];
+ if (!brands) return { pending: [], retry: [] };
+ const pending: AiPendingTask[] = [];
+ const retry: AiPendingTask[] = [];
for (let i = 0; i < tasks.length; i += TASK_GET_CONCURRENCY) {
const chunk = tasks.slice(i, i + TASK_GET_CONCURRENCY);
const outcomes = await Promise.allSettled(
@@ -283,13 +346,23 @@ export async function collectAiRound(
const task = chunk[index];
// A failed read is retried next round.
if (outcome.status === "rejected" || outcome.value.status === "pending") {
- stillPending.push(task);
+ pending.push(task);
+ continue;
+ }
+ const result = outcome.value;
+ if (
+ canRetry &&
+ result.status === "failed" &&
+ result.retryable &&
+ task.attempts < AI_ANSWER_ATTEMPTS
+ ) {
+ retry.push(task);
continue;
}
- await saveAiOutcome(runId, task, outcome.value, brands);
+ await saveAiOutcome(runId, task, result, brands);
}
}
- return stillPending;
+ return { pending, retry: engineBatches(retry, MAX_TASKS_PER_POST) };
}
/** Brands come from the project as each answer is matched. */
diff --git a/src/server/lib/dataforseo/ai-tracking.ts b/src/server/lib/dataforseo/ai-tracking.ts
index 7b7bcd2d..eb68a98c 100644
--- a/src/server/lib/dataforseo/ai-tracking.ts
+++ b/src/server/lib/dataforseo/ai-tracking.ts
@@ -3,6 +3,7 @@ import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared";
import {
isNoResultsTask,
isTaskInProgress,
+ isUpstreamFailureStatus,
type DataforseoApiResponse,
type DataforseoTaskLike,
failedResponseError,
@@ -140,7 +141,12 @@ export async function postAiTrackingTasks(
}
export type AiTrackingAnswer =
- | { status: "failed"; message: string }
+ | {
+ status: "failed";
+ message: string;
+ /** DataForSEO's own backend failed the task, so a new one can succeed. */
+ retryable: boolean;
+ }
/** `result` is null when Google returned no results page. */
| { status: "completed"; result: unknown };
type AiTrackingTaskOutcome = { status: "pending" } | AiTrackingAnswer;
@@ -173,6 +179,7 @@ function aiTrackingAnswer(
status: "failed",
message:
task.status_message || `DataForSEO task failed (${task.status_code})`,
+ retryable: isUpstreamFailureStatus(task.status_code),
};
}
return { status: "completed", result: task.result?.[0] ?? null };
diff --git a/src/server/lib/dataforseo/ai.test.ts b/src/server/lib/dataforseo/ai.test.ts
new file mode 100644
index 00000000..4d7d3b3d
--- /dev/null
+++ b/src/server/lib/dataforseo/ai.test.ts
@@ -0,0 +1,26 @@
+import { expect, it, vi } from "vitest";
+import { fetchLlmResponse } from "@/server/lib/dataforseo/ai";
+
+vi.mock("@/server/lib/runtime-env", () => ({
+ getRequiredEnvValue: vi.fn(async () => "test-api-key"),
+}));
+vi.mock("@/server/lib/dataforseo/llm-models", () => ({
+ isKnownLlmModelName: async () => true,
+}));
+
+it("never replays a billed llm_responses call on a 5xx (a retry could double-charge)", async () => {
+ const fetchMock = vi
+ .fn<typeof fetch>()
+ .mockResolvedValue(new Response("upstream error", { status: 502 }));
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(
+ fetchLlmResponse({
+ modelSlug: "claude",
+ modelName: "claude-sonnet-5",
+ userPrompt: "Which SEO tools?",
+ webSearch: false,
+ }),
+ ).rejects.toMatchObject({ code: "UPSTREAM_UNAVAILABLE" });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+});
diff --git a/src/server/lib/dataforseo/ai.ts b/src/server/lib/dataforseo/ai.ts
index 9e3f99eb..a6dfe648 100644
--- a/src/server/lib/dataforseo/ai.ts
+++ b/src/server/lib/dataforseo/ai.ts
@@ -329,7 +329,8 @@ export async function fetchLlmResponse(
const response = await dataforseoPost(
`/v3/ai_optimization/${input.modelSlug}/llm_responses/live`,
[fields],
- { classify: classifyAiSearchError },
+ // A billed live call must never be replayed on an ambiguous 5xx.
+ { classify: classifyAiSearchError, maxServerErrorRetries: 0 },
);
const task = assertOk(
diff --git a/src/server/lib/dataforseo/envelope.ts b/src/server/lib/dataforseo/envelope.ts
index e9884b12..ff0f7050 100644
--- a/src/server/lib/dataforseo/envelope.ts
+++ b/src/server/lib/dataforseo/envelope.ts
@@ -174,7 +174,9 @@ const UPSTREAM_FAILURE_STATUS_CODES = new Set([
50402, // Target page took too long to respond.
]);
-function isUpstreamFailureStatus(statusCode: number | undefined): boolean {
+export function isUpstreamFailureStatus(
+ statusCode: number | undefined,
+): boolean {
return (
statusCode !== undefined && UPSTREAM_FAILURE_STATUS_CODES.has(statusCode)
);
diff --git a/src/server/workflows/AiVisibilityWorkflow.test.ts b/src/server/workflows/AiVisibilityWorkflow.test.ts
index cb8161f8..05b120e7 100644
--- a/src/server/workflows/AiVisibilityWorkflow.test.ts
+++ b/src/server/workflows/AiVisibilityWorkflow.test.ts
@@ -73,7 +73,7 @@ beforeEach(() => {
],
});
mocks.post.mockResolvedValue([task]);
- mocks.collect.mockResolvedValue([]);
+ mocks.collect.mockResolvedValue({ pending: [], retry: [] });
});
describe("AI visibility workflow", () => {
@@ -84,7 +84,11 @@ describe("AI visibility workflow", () => {
retries: { limit: 0, delay: "1 second" },
timeout: "2 minutes",
});
- expect(mocks.collect).toHaveBeenCalledExactlyOnceWith("run", [task, task]);
+ expect(mocks.collect).toHaveBeenCalledExactlyOnceWith(
+ "run",
+ [task, task],
+ true,
+ );
expect(mocks.sleep).toHaveBeenCalledOnce();
expect(mocks.finalize).toHaveBeenCalledExactlyOnceWith("run");
});
@@ -98,7 +102,7 @@ describe("AI visibility workflow", () => {
});
await execute();
expect(mocks.model).toHaveBeenCalledOnce();
- expect(mocks.steps).toHaveBeenCalledWith("collect-model-0", {
+ expect(mocks.steps).toHaveBeenCalledWith("read-model-0", {
retries: { limit: 2, delay: "10 seconds" },
timeout: "5 minutes",
});
@@ -121,12 +125,33 @@ describe("AI visibility workflow", () => {
});
it("stops polling when the collection window closes and finalizes what arrived", async () => {
- mocks.collect.mockResolvedValue([task]);
+ mocks.collect.mockResolvedValue({ pending: [task], retry: [] });
await execute();
expect(mocks.sleep).toHaveBeenCalledTimes(9);
expect(mocks.finalize).toHaveBeenCalledOnce();
});
+ it("posts failed answers again in a step that never retries, but not after the last round", async () => {
+ const batch = { engine: "chatgpt", tasks: [{ tag: "answer", prompt: "" }] };
+ mocks.collect.mockResolvedValue({ pending: [task], retry: [batch] });
+ await execute();
+ expect(mocks.post).toHaveBeenCalledWith(
+ "run",
+ customer,
+ expect.anything(),
+ batch,
+ );
+ expect(mocks.steps).toHaveBeenCalledWith("repost-0-0", {
+ retries: { limit: 0, delay: "1 second" },
+ timeout: "2 minutes",
+ });
+ expect(mocks.collect).toHaveBeenLastCalledWith(
+ "run",
+ expect.anything(),
+ false,
+ );
+ });
+
it("fails the run when preparation fails, such as when credits run out", async () => {
const error = new Error("Not enough credits");
mocks.prepare.mockRejectedValue(error);
diff --git a/src/server/workflows/AiVisibilityWorkflow.ts b/src/server/workflows/AiVisibilityWorkflow.ts
index 83ec85ee..b751ed55 100644
--- a/src/server/workflows/AiVisibilityWorkflow.ts
+++ b/src/server/workflows/AiVisibilityWorkflow.ts
@@ -38,8 +38,9 @@ const COLLECT_STEP = {
timeout: "5 minutes" as const,
};
// Live answers are billed per call, so a live step runs once too. It waits for
-// up to 6 answers, each up to 120 seconds.
-const LIVE_STEP = { ...SINGLE_ATTEMPT, timeout: "5 minutes" as const };
+// up to 6 answers, each up to 120 seconds, and asks a failed answer up to 3
+// times.
+const LIVE_STEP = { ...SINGLE_ATTEMPT, timeout: "10 minutes" as const };
// Standard-queue answers usually arrive within minutes. Cumulative waits are
// 2 / 4 / 7 / 10 / 14 / 18 / 22 / 26 / 30 minutes; anything still missing then
// fails as not collected. Each sleep stays under 5 minutes, like rank check's:
@@ -88,6 +89,39 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<
};
let pending: AiPendingTask[] = [];
let plan: Awaited<ReturnType<typeof prepareAiRun>>;
+ // Reads the pending answers once, then posts each answer that DataForSEO's
+ // backend failed again in its own step, since a post is billed. The read
+ // steps are `read-*`, not the older `collect-*`: a run in flight across a
+ // deploy must not replay a `collect-*` output, which had another shape.
+ const collect = async (name: string, canRetry: boolean) => {
+ const remaining = pending;
+ let round: Awaited<ReturnType<typeof collectAiRound>>;
+ try {
+ round = await pgStep(step, `read-${name}`, COLLECT_STEP, () =>
+ collectAiRound(runId, remaining, canRetry),
+ );
+ } catch (error) {
+ // Posted tasks are already paid for; keep polling them next round.
+ console.warn(`[ai-visibility] ${runId} read-${name}:`, error);
+ return;
+ }
+ pending = round.pending;
+ for (const [index, batch] of round.retry.entries()) {
+ try {
+ pending.push(
+ ...(await pgStep(
+ step,
+ `repost-${name}-${index}`,
+ SINGLE_ATTEMPT,
+ () => postAiBatch(runId, customer, plan.market, batch),
+ )),
+ );
+ } catch (error) {
+ // Its answers fail at finalize.
+ console.warn(`[ai-visibility] ${runId} repost-${name}:`, error);
+ }
+ }
+ };
try {
plan = await pgStep(step, "prepare", SINGLE_ATTEMPT, () =>
prepareAiRun(runId, customer),
@@ -121,18 +155,7 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<
// Its unsaved answers fail at finalize; the other batches continue.
console.warn(`[ai-visibility] ${runId} model-${index}:`, error);
}
- if (!pending.length) continue;
- const remaining = pending;
- try {
- pending = await pgStep(
- step,
- `collect-model-${index}`,
- COLLECT_STEP,
- () => collectAiRound(runId, remaining),
- );
- } catch (error) {
- console.warn(`[ai-visibility] ${runId} collect-model-${index}:`, error);
- }
+ if (pending.length) await collect(`model-${index}`, true);
}
for (
let round = 0;
@@ -148,15 +171,8 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<
// log it as one.
return markFailed(error);
}
- const remaining = pending;
- try {
- pending = await pgStep(step, `collect-${round}`, COLLECT_STEP, () =>
- collectAiRound(runId, remaining),
- );
- } catch (error) {
- // Posted tasks are already paid for; keep polling them next round.
- console.warn(`[ai-visibility] ${runId} collect-${round}:`, error);
- }
+ // A task posted after the last round would never be read.
+ await collect(`${round}`, round < POLL_INTERVALS.length - 1);
}
try {
await pgStep(step, "finalize", SINGLE_ATTEMPT, () =>