Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Retry AI visibility answers that DataForSEO's backend fails, up to 3 attempts (EVE-419) (#926)
Carter Smith95a5f3f2cdb4
diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.tsindex 7971d282..ad61d34f 100644--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.test.ts@@ -1,5 +1,7 @@-import { beforeEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { aiObservations, projectCompetitors } from "@/db/schema";+import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";+import { AppError } from "@/server/lib/errors";import { AiVisibilityRepository as repo } from "../repositories/AiVisibilityRepository";import {collectAiLiveBatch,@@ -46,6 +48,26 @@ vi.mock("@/server/lib/dataforseo/llm-models", () => ({const runId = "run";const market = { locationCode: 2840, languageCode: "en" };+const pendingTask = {+ tag: "answer-0",+ taskId: "task",+ engine: "chatgpt",+ prompt: "Which SEO tools?",+ attempts: 1,+} as const;+const liveAnswer = {+ status: "fulfilled",+ value: { status: "completed", result: { markdown: "OpenSEO", sources: [] } },+} as const;+const providerFailure = {+ status: "failed",+ message: "Internal Server Error.",+ retryable: true,+} as const;++afterEach(() => {+ vi.useRealTimers();+});beforeEach(async () => {await testDb.seedProject();@@ -97,11 +119,9 @@ describe("AI answer collection", () => {},});- const pending = await collectAiRound(runId, [- { tag: "answer-0", taskId: "task", engine: "chatgpt" },- ]);+ const round = await collectAiRound(runId, [pendingTask], true);- expect(pending).toEqual([]);+ expect(round).toEqual({ pending: [], retry: [] });expect(await repo.getObservation("answer-0")).toMatchObject({status: "completed",});@@ -142,11 +162,51 @@ describe("AI answer collection", () => {],});- expect(pending).toEqual([- { tag: "answer-0", taskId: "task", engine: "chatgpt" },+ expect(pending).toEqual([pendingTask]);+ expect(await repo.getObservation("answer-1")).toMatchObject({+ status: "failed",+ });+ });++ it("counts the attempt of an answer posted again, so its retries stop at 3", async () => {+ dataforseo.trackingTaskPost.mockResolvedValue([+ { tag: "answer-0", taskId: "task" },+ ]);++ const pending = await postAiBatch(runId, customer, market, {+ engine: "chatgpt",+ tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 2 }],+ });++ expect(pending).toEqual([{ ...pendingTask, attempts: 3 }]);+ });++ it("returns a queued answer DataForSEO's backend failed for a new post, and fails it after the last attempt", async () => {+ dataforseo.fetchTaskResult.mockResolvedValue(providerFailure);++ const round = await collectAiRound(+ runId,+ [+ pendingTask,+ {+ ...pendingTask,+ tag: "answer-1",+ prompt: "Best rank tracker?",+ attempts: 3,+ },+ ],+ true,+ );++ expect(round.retry).toEqual([+ {+ engine: "chatgpt",+ tasks: [{ tag: "answer-0", prompt: "Which SEO tools?", attempts: 1 }],+ },]);expect(await repo.getObservation("answer-1")).toMatchObject({status: "failed",+ error: "Internal Server Error.",});});@@ -226,6 +286,42 @@ describe("AI answer collection", () => {});});+ it("asks a model answer again only when DataForSEO returned the failed task", async () => {+ vi.useFakeTimers({ toFake: ["setTimeout"] });+ dataforseo.llmResponse.mockImplementation(+ async ({ userPrompt }: { userPrompt: string }) => {+ if (userPrompt === "Best rank tracker?")+ throw new AppError("UPSTREAM_UNAVAILABLE", "DataForSEO HTTP 502");+ throw new DataforseoChargedTaskError(+ "Internal Server Error.",+ { path: [], costUsd: 0 },+ false,+ "UPSTREAM_UNAVAILABLE",+ );+ },+ );++ const collecting = collectAiModelBatch(runId, customer, market, {+ engine: "claude",+ tasks: [+ { tag: "answer-0", prompt: "Which SEO tools?" },+ { tag: "answer-1", prompt: "Best rank tracker?" },+ ],+ });+ await vi.runAllTimersAsync();+ await collecting;++ expect(dataforseo.llmResponse).toHaveBeenCalledTimes(4);+ expect(await repo.getObservation("answer-0")).toMatchObject({+ status: "failed",+ error: "Internal Server Error.",+ });+ expect(await repo.getObservation("answer-1")).toMatchObject({+ status: "failed",+ error: "DataForSEO HTTP 502",+ });+ });+it("sends Claude to the model API, not the live scraper, on a manual run", async () => {const [{ promptId }] = await repo.getObservations([runId]);await testDb.db.insert(aiObservations).values({@@ -249,13 +345,7 @@ describe("AI answer collection", () => {it("collects a manual run live, saving answers and failing rejected calls", async () => {expect(await prepareAiRun(runId, customer)).toMatchObject({ live: true });dataforseo.trackingLiveBatch.mockResolvedValue([- {- status: "fulfilled",- value: {- status: "completed",- result: { markdown: "OpenSEO", sources: [] },- },- },+ liveAnswer,{ status: "rejected", reason: new Error("Timed out") },]);@@ -271,17 +361,39 @@ describe("AI answer collection", () => {status: "failed",error: "Timed out",});+ expect(dataforseo.trackingLiveBatch).toHaveBeenCalledOnce();});- it("saves the rest of a paid live batch when one answer cannot be saved", async () => {- const answer = {+ it("asks a live answer again when DataForSEO's backend fails it, up to 3 attempts", async () => {+ vi.useFakeTimers({ toFake: ["setTimeout"] });+ const failed = (message: string) => ({status: "fulfilled",- value: {- status: "completed",- result: { markdown: "OpenSEO", sources: [] },- },- };- dataforseo.trackingLiveBatch.mockResolvedValue([answer, answer]);+ value: { ...providerFailure, message },+ });+ dataforseo.trackingLiveBatch+ .mockResolvedValueOnce([failed("Internal Server Error."), failed("1")])+ .mockResolvedValueOnce([liveAnswer, failed("2")])+ .mockResolvedValueOnce([failed("Internal SE Server Error.")]);++ const collecting = collectAiLiveBatch(runId, customer, market, [+ { tag: "answer-0", prompt: "Which SEO tools?", engine: "chatgpt" },+ { tag: "answer-1", prompt: "Best rank tracker?", engine: "chatgpt" },+ ]);+ await vi.runAllTimersAsync();+ await collecting;++ expect(dataforseo.trackingLiveBatch).toHaveBeenCalledTimes(3);+ expect(await repo.getObservation("answer-0")).toMatchObject({+ status: "completed",+ });+ expect(await repo.getObservation("answer-1")).toMatchObject({+ status: "failed",+ error: "Internal SE Server Error.",+ });+ });++ it("saves the rest of a paid live batch when one answer cannot be saved", async () => {+ dataforseo.trackingLiveBatch.mockResolvedValue([liveAnswer, liveAnswer]);vi.spyOn(repo, "persistAnswer").mockRejectedValueOnce(new Error("D1"));vi.spyOn(console, "error").mockImplementation(() => {});@@ -303,9 +415,7 @@ describe("AI answer collection", () => {status: "completed",result: { markdown: "OpenSEO", sources: [] },});- await collectAiRound(runId, [- { tag: "answer-0", taskId: "task", engine: "chatgpt" },- ]);+ await collectAiRound(runId, [pendingTask], true);await finalizeAiRun(runId);diff --git a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts b/src/server/features/ai-visibility/services/aiVisibilityCollection.tsindex 50426e0d..d28f4a1a 100644--- a/src/server/features/ai-visibility/services/aiVisibilityCollection.ts+++ b/src/server/features/ai-visibility/services/aiVisibilityCollection.ts@@ -7,6 +7,7 @@ import {type AiTrackingAnswer,type PostedAiTrackingTask,} from "@/server/lib/dataforseo";+import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";import { resolveLatestLlmModelName } from "@/server/lib/dataforseo/llm-models";import { AppError } from "@/server/lib/errors";import { isHostedServerAuthMode } from "@/server/lib/runtime-env";@@ -30,18 +31,22 @@ import { failAiRun } from "./aiVisibilityRuns";// Each function here is one workflow step body: inputs are its parameters and// the return value is what the workflow engine persists and replays.-interface AiTaskBatch {- engine: AiEngine;- /** Tag = answer row id. At most 100 per task_post, 6 per model batch. */- tasks: { tag: string; prompt: string }[];-}-/** One live answer. A live step mixes engines, so each task names its own. */-interface AiLiveTask {+interface AiTask {+ /** The answer row id. */tag: string;prompt: string;+ /** Attempts already made, when a queued answer is posted again. */+ attempts?: number;+}+interface AiTaskBatch {engine: AiEngine;+ /** At most 100 per task_post, 6 per model batch. */+ tasks: AiTask[];}-export type AiPendingTask = PostedAiTrackingTask & { engine: AiEngine };+/** One live answer. A live step mixes engines, so each task names its own. */+type AiLiveTask = AiTask & { engine: AiEngine };+export type AiPendingTask = PostedAiTrackingTask &+ AiLiveTask & { attempts: number };interface AiRunMarket {locationCode: number;languageCode: string;@@ -53,6 +58,22 @@ interface AiRunMarket {*/const LIVE_ANSWERS_PER_STEP = 6;+/**+ * Attempts per answer. A new attempt follows only when DataForSEO returned the+ * task and its own backend failed it, so the failure and its cost are known. A+ * transport error or a timeout can hide a charge, so it is never retried.+ */+const AI_ANSWER_ATTEMPTS = 3;+/** Pause before a new attempt, so a short provider outage can clear. */+const AI_RETRY_DELAY_MS = 3_000;++const waitBeforeRetry = () =>+ new Promise((resolve) => setTimeout(resolve, AI_RETRY_DELAY_MS));++function errorMessage(error: unknown) {+ return error instanceof Error ? error.message : String(error);+}+/*** Marks the run running and groups its answers into batches. Manual and* baseline runs collect live answers so the user sees results sooner.@@ -94,20 +115,20 @@ export async function prepareAiRun(locationCode: run.locationCode,languageCode: run.languageCode,};+ const tasks = pending.map((row) => ({+ tag: row.id,+ prompt: row.prompt,+ engine: row.engine,+ }));const modelBatches = engineBatches(- pending.filter((row) => aiEngineUsesModelApi(row.engine)),+ tasks.filter((task) => aiEngineUsesModelApi(task.engine)),LIVE_ANSWERS_PER_STEP,);- const answers = pending.filter((row) => !aiEngineUsesModelApi(row.engine));+ const answers = tasks.filter((task) => !aiEngineUsesModelApi(task.engine));if (live) {- const tasks = answers.map((row) => ({- tag: row.id,- prompt: row.prompt,- engine: row.engine,- }));const batches: AiLiveTask[][] = [];- for (let i = 0; i < tasks.length; i += LIVE_ANSWERS_PER_STEP)- batches.push(tasks.slice(i, i + LIVE_ANSWERS_PER_STEP));+ for (let i = 0; i < answers.length; i += LIVE_ANSWERS_PER_STEP)+ batches.push(answers.slice(i, i + LIVE_ANSWERS_PER_STEP));return { market, modelBatches, live, batches };}return {@@ -119,15 +140,12 @@ export async function prepareAiRun(}/** Groups answers by engine into batches of at most `size`. */-function engineBatches(- answers: { id: string; prompt: string; engine: AiEngine }[],- size: number,-): AiTaskBatch[] {+function engineBatches(answers: AiLiveTask[], size: number): AiTaskBatch[] {const batches: AiTaskBatch[] = [];- for (const engine of new Set(answers.map((row) => row.engine))) {+ for (const engine of new Set(answers.map((task) => task.engine))) {const tasks = answers- .filter((row) => row.engine === engine)- .map((row) => ({ tag: row.id, prompt: row.prompt }));+ .filter((task) => task.engine === engine)+ .map(({ tag, prompt, attempts }) => ({ tag, prompt, attempts }));for (let i = 0; i < tasks.length; i += size)batches.push({ engine, tasks: tasks.slice(i, i + size) });}@@ -153,20 +171,26 @@ export async function postAiBatch(...market,});} catch (cause) {- error = cause instanceof Error ? cause.message : String(cause);+ error = errorMessage(cause);}- const accepted = new Set(posted.map((task) => task.tag));+ const taskIds = new Map(posted.map((task) => [task.tag, task.taskId]));const rejected = batch.tasks.map((task) => task.tag)- .filter((tag) => !accepted.has(tag));+ .filter((tag) => !taskIds.has(tag));if (rejected.length)await repo.failPendingObservations(runId, error, rejected);- return posted.map((task) => ({ ...task, engine: batch.engine }));+ return batch.tasks.flatMap((task) => {+ const taskId = taskIds.get(task.tag);+ if (!taskId) return [];+ const attempts = (task.attempts ?? 0) + 1;+ return [{ ...task, taskId, engine: batch.engine, attempts }];+ });}/*** Collects one batch of live answers through the metered client, which bills- * them, and saves each one. Answers that fail now are not retried.+ * them, and saves each one. An answer that DataForSEO's backend failed is+ * asked again, up to AI_ANSWER_ATTEMPTS.*/export async function collectAiLiveBatch(runId: string,@@ -176,6 +200,43 @@ export async function collectAiLiveBatch() {const brands = await runBrands(runId);if (!brands) return;+ let remaining = tasks;+ for (let attempt = 1; remaining.length > 0; attempt++) {+ if (attempt > 1) await waitBeforeRetry();+ const answers = await fetchLiveAnswers(customer, market, remaining);+ const retry: AiLiveTask[] = [];+ for (const [index, result] of answers.entries()) {+ const task = remaining[index];+ if (+ result.status === "failed" &&+ result.retryable &&+ attempt < AI_ANSWER_ATTEMPTS+ ) {+ retry.push(task);+ continue;+ }+ // Every answer here is already paid for, and the step cannot rerun, so+ // one failed write must not drop the rest. finalizeAiRun fails the+ // answer that stays pending.+ try {+ await saveAiOutcome(runId, task, result, brands);+ } catch (error) {+ console.error(+ `[ai-visibility] ${runId} could not save answer ${task.tag}:`,+ error,+ );+ }+ }+ remaining = retry;+ }+}++/** A call that throws becomes a failed answer that is not retried. */+async function fetchLiveAnswers(+ customer: BillingCustomerContext,+ market: AiRunMarket,+ tasks: AiLiveTask[],+): Promise<AiTrackingAnswer[]> {let settled: PromiseSettledResult<AiTrackingAnswer>[];try {settled = await createDataforseoClient(customer).aiSearch.trackingLiveBatch(@@ -184,37 +245,24 @@ export async function collectAiLiveBatch(} catch (error) {settled = tasks.map(() => ({ status: "rejected", reason: error }));}- for (const [index, outcome] of settled.entries()) {- const result: AiTrackingAnswer =- outcome.status === "fulfilled"- ? outcome.value- : {- status: "failed",- message:- outcome.reason instanceof Error- ? outcome.reason.message- : String(outcome.reason),- };- // Every answer here is already paid for, and the step cannot rerun, so- // one failed write must not drop the rest. finalizeAiRun fails the- // answer that stays pending.- try {- await saveAiOutcome(runId, tasks[index], result, brands);- } catch (error) {- console.error(- `[ai-visibility] ${runId} could not save answer ${tasks[index].tag}:`,- error,- );- }- }+ return settled.map((outcome) =>+ outcome.status === "fulfilled"+ ? outcome.value+ : {+ status: "failed",+ message: errorMessage(outcome.reason),+ retryable: false,+ },+ );}/*** Asks each prompt of a Claude or Perplexity batch through the metered model* API client, which bills each answer, and saves each answer as soon as it* arrives, so a slow call that hits the step timeout cannot discard answers- * already paid for. Model answers have no language setting: only the country- * guides the web search.+ * already paid for. An answer that DataForSEO's backend failed is asked again,+ * up to AI_ANSWER_ATTEMPTS. Model answers have no language setting: only the+ * country guides the web search.*/export async function collectAiModelBatch(runId: string,@@ -232,23 +280,34 @@ export async function collectAiModelBatch(const country = getIsoCountryCode(market.locationCode).toUpperCase();await Promise.all(batch.tasks.map(async (task) => {- try {- const result = await client.aiSearch.llmResponse({- modelSlug,- modelName,- userPrompt: task.prompt,- webSearch: true,- // Saving the tracker rejects countries the engine cannot search.- webSearchCountryCode: country,- maxOutputTokens: MODEL_MAX_OUTPUT_TOKENS,- });- await saveAiAnswer(task.tag, parseLlmResponseAnswer(result), brands);- } catch (cause) {- await repo.failPendingObservations(- runId,- cause instanceof Error ? cause.message : String(cause),- [task.tag],- );+ for (let attempt = 1; ; attempt++) {+ try {+ const result = await client.aiSearch.llmResponse({+ modelSlug,+ modelName,+ userPrompt: task.prompt,+ webSearch: true,+ // Saving the tracker rejects countries the engine cannot search.+ webSearchCountryCode: country,+ maxOutputTokens: MODEL_MAX_OUTPUT_TOKENS,+ });+ await saveAiAnswer(task.tag, parseLlmResponseAnswer(result), brands);+ return;+ } catch (cause) {+ // A charged task error is a task DataForSEO returned with its cost.+ if (+ cause instanceof DataforseoChargedTaskError &&+ cause.code === "UPSTREAM_UNAVAILABLE" &&+ attempt < AI_ANSWER_ATTEMPTS+ ) {+ await waitBeforeRetry();+ continue;+ }+ await repo.failPendingObservations(runId, errorMessage(cause), [+ task.tag,+ ]);+ return;+ }}}),);@@ -265,15 +324,19 @@ const TASK_GET_CONCURRENCY = 25;/*** Reads each pending answer once (task_get is free) and saves finished ones- * with their citations and brand results. Returns the tasks still pending.+ * with their citations and brand results. Returns the tasks still pending, and+ * the answers DataForSEO's backend failed that `canRetry` lets the caller post+ * again.*/export async function collectAiRound(runId: string,tasks: AiPendingTask[],-): Promise<AiPendingTask[]> {+ canRetry: boolean,+): Promise<{ pending: AiPendingTask[]; retry: AiTaskBatch[] }> {const brands = await runBrands(runId);- if (!brands) return [];- const stillPending: AiPendingTask[] = [];+ if (!brands) return { pending: [], retry: [] };+ const pending: AiPendingTask[] = [];+ const retry: AiPendingTask[] = [];for (let i = 0; i < tasks.length; i += TASK_GET_CONCURRENCY) {const chunk = tasks.slice(i, i + TASK_GET_CONCURRENCY);const outcomes = await Promise.allSettled(@@ -283,13 +346,23 @@ export async function collectAiRound(const task = chunk[index];// A failed read is retried next round.if (outcome.status === "rejected" || outcome.value.status === "pending") {- stillPending.push(task);+ pending.push(task);+ continue;+ }+ const result = outcome.value;+ if (+ canRetry &&+ result.status === "failed" &&+ result.retryable &&+ task.attempts < AI_ANSWER_ATTEMPTS+ ) {+ retry.push(task);continue;}- await saveAiOutcome(runId, task, outcome.value, brands);+ await saveAiOutcome(runId, task, result, brands);}}- return stillPending;+ return { pending, retry: engineBatches(retry, MAX_TASKS_PER_POST) };}/** Brands come from the project as each answer is matched. */diff --git a/src/server/lib/dataforseo/ai-tracking.ts b/src/server/lib/dataforseo/ai-tracking.tsindex 7b7bcd2d..eb68a98c 100644--- a/src/server/lib/dataforseo/ai-tracking.ts+++ b/src/server/lib/dataforseo/ai-tracking.ts@@ -3,6 +3,7 @@ import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared";import {isNoResultsTask,isTaskInProgress,+ isUpstreamFailureStatus,type DataforseoApiResponse,type DataforseoTaskLike,failedResponseError,@@ -140,7 +141,12 @@ export async function postAiTrackingTasks(}export type AiTrackingAnswer =- | { status: "failed"; message: string }+ | {+ status: "failed";+ message: string;+ /** DataForSEO's own backend failed the task, so a new one can succeed. */+ retryable: boolean;+ }/** `result` is null when Google returned no results page. */| { status: "completed"; result: unknown };type AiTrackingTaskOutcome = { status: "pending" } | AiTrackingAnswer;@@ -173,6 +179,7 @@ function aiTrackingAnswer(status: "failed",message:task.status_message || `DataForSEO task failed (${task.status_code})`,+ retryable: isUpstreamFailureStatus(task.status_code),};}return { status: "completed", result: task.result?.[0] ?? null };diff --git a/src/server/lib/dataforseo/ai.test.ts b/src/server/lib/dataforseo/ai.test.tsnew file mode 100644index 00000000..4d7d3b3d--- /dev/null+++ b/src/server/lib/dataforseo/ai.test.ts@@ -0,0 +1,26 @@+import { expect, it, vi } from "vitest";+import { fetchLlmResponse } from "@/server/lib/dataforseo/ai";++vi.mock("@/server/lib/runtime-env", () => ({+ getRequiredEnvValue: vi.fn(async () => "test-api-key"),+}));+vi.mock("@/server/lib/dataforseo/llm-models", () => ({+ isKnownLlmModelName: async () => true,+}));++it("never replays a billed llm_responses call on a 5xx (a retry could double-charge)", async () => {+ const fetchMock = vi+ .fn<typeof fetch>()+ .mockResolvedValue(new Response("upstream error", { status: 502 }));+ vi.stubGlobal("fetch", fetchMock);++ await expect(+ fetchLlmResponse({+ modelSlug: "claude",+ modelName: "claude-sonnet-5",+ userPrompt: "Which SEO tools?",+ webSearch: false,+ }),+ ).rejects.toMatchObject({ code: "UPSTREAM_UNAVAILABLE" });+ expect(fetchMock).toHaveBeenCalledTimes(1);+});diff --git a/src/server/lib/dataforseo/ai.ts b/src/server/lib/dataforseo/ai.tsindex 9e3f99eb..a6dfe648 100644--- a/src/server/lib/dataforseo/ai.ts+++ b/src/server/lib/dataforseo/ai.ts@@ -329,7 +329,8 @@ export async function fetchLlmResponse(const response = await dataforseoPost(`/v3/ai_optimization/${input.modelSlug}/llm_responses/live`,[fields],- { classify: classifyAiSearchError },+ // A billed live call must never be replayed on an ambiguous 5xx.+ { classify: classifyAiSearchError, maxServerErrorRetries: 0 },);const task = assertOk(diff --git a/src/server/lib/dataforseo/envelope.ts b/src/server/lib/dataforseo/envelope.tsindex e9884b12..ff0f7050 100644--- a/src/server/lib/dataforseo/envelope.ts+++ b/src/server/lib/dataforseo/envelope.ts@@ -174,7 +174,9 @@ const UPSTREAM_FAILURE_STATUS_CODES = new Set([50402, // Target page took too long to respond.]);-function isUpstreamFailureStatus(statusCode: number | undefined): boolean {+export function isUpstreamFailureStatus(+ statusCode: number | undefined,+): boolean {return (statusCode !== undefined && UPSTREAM_FAILURE_STATUS_CODES.has(statusCode));diff --git a/src/server/workflows/AiVisibilityWorkflow.test.ts b/src/server/workflows/AiVisibilityWorkflow.test.tsindex cb8161f8..05b120e7 100644--- a/src/server/workflows/AiVisibilityWorkflow.test.ts+++ b/src/server/workflows/AiVisibilityWorkflow.test.ts@@ -73,7 +73,7 @@ beforeEach(() => {],});mocks.post.mockResolvedValue([task]);- mocks.collect.mockResolvedValue([]);+ mocks.collect.mockResolvedValue({ pending: [], retry: [] });});describe("AI visibility workflow", () => {@@ -84,7 +84,11 @@ describe("AI visibility workflow", () => {retries: { limit: 0, delay: "1 second" },timeout: "2 minutes",});- expect(mocks.collect).toHaveBeenCalledExactlyOnceWith("run", [task, task]);+ expect(mocks.collect).toHaveBeenCalledExactlyOnceWith(+ "run",+ [task, task],+ true,+ );expect(mocks.sleep).toHaveBeenCalledOnce();expect(mocks.finalize).toHaveBeenCalledExactlyOnceWith("run");});@@ -98,7 +102,7 @@ describe("AI visibility workflow", () => {});await execute();expect(mocks.model).toHaveBeenCalledOnce();- expect(mocks.steps).toHaveBeenCalledWith("collect-model-0", {+ expect(mocks.steps).toHaveBeenCalledWith("read-model-0", {retries: { limit: 2, delay: "10 seconds" },timeout: "5 minutes",});@@ -121,12 +125,33 @@ describe("AI visibility workflow", () => {});it("stops polling when the collection window closes and finalizes what arrived", async () => {- mocks.collect.mockResolvedValue([task]);+ mocks.collect.mockResolvedValue({ pending: [task], retry: [] });await execute();expect(mocks.sleep).toHaveBeenCalledTimes(9);expect(mocks.finalize).toHaveBeenCalledOnce();});+ it("posts failed answers again in a step that never retries, but not after the last round", async () => {+ const batch = { engine: "chatgpt", tasks: [{ tag: "answer", prompt: "" }] };+ mocks.collect.mockResolvedValue({ pending: [task], retry: [batch] });+ await execute();+ expect(mocks.post).toHaveBeenCalledWith(+ "run",+ customer,+ expect.anything(),+ batch,+ );+ expect(mocks.steps).toHaveBeenCalledWith("repost-0-0", {+ retries: { limit: 0, delay: "1 second" },+ timeout: "2 minutes",+ });+ expect(mocks.collect).toHaveBeenLastCalledWith(+ "run",+ expect.anything(),+ false,+ );+ });+it("fails the run when preparation fails, such as when credits run out", async () => {const error = new Error("Not enough credits");mocks.prepare.mockRejectedValue(error);diff --git a/src/server/workflows/AiVisibilityWorkflow.ts b/src/server/workflows/AiVisibilityWorkflow.tsindex 83ec85ee..b751ed55 100644--- a/src/server/workflows/AiVisibilityWorkflow.ts+++ b/src/server/workflows/AiVisibilityWorkflow.ts@@ -38,8 +38,9 @@ const COLLECT_STEP = {timeout: "5 minutes" as const,};// Live answers are billed per call, so a live step runs once too. It waits for-// up to 6 answers, each up to 120 seconds.-const LIVE_STEP = { ...SINGLE_ATTEMPT, timeout: "5 minutes" as const };+// up to 6 answers, each up to 120 seconds, and asks a failed answer up to 3+// times.+const LIVE_STEP = { ...SINGLE_ATTEMPT, timeout: "10 minutes" as const };// Standard-queue answers usually arrive within minutes. Cumulative waits are// 2 / 4 / 7 / 10 / 14 / 18 / 22 / 26 / 30 minutes; anything still missing then// fails as not collected. Each sleep stays under 5 minutes, like rank check's:@@ -88,6 +89,39 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<};let pending: AiPendingTask[] = [];let plan: Awaited<ReturnType<typeof prepareAiRun>>;+ // Reads the pending answers once, then posts each answer that DataForSEO's+ // backend failed again in its own step, since a post is billed. The read+ // steps are `read-*`, not the older `collect-*`: a run in flight across a+ // deploy must not replay a `collect-*` output, which had another shape.+ const collect = async (name: string, canRetry: boolean) => {+ const remaining = pending;+ let round: Awaited<ReturnType<typeof collectAiRound>>;+ try {+ round = await pgStep(step, `read-${name}`, COLLECT_STEP, () =>+ collectAiRound(runId, remaining, canRetry),+ );+ } catch (error) {+ // Posted tasks are already paid for; keep polling them next round.+ console.warn(`[ai-visibility] ${runId} read-${name}:`, error);+ return;+ }+ pending = round.pending;+ for (const [index, batch] of round.retry.entries()) {+ try {+ pending.push(+ ...(await pgStep(+ step,+ `repost-${name}-${index}`,+ SINGLE_ATTEMPT,+ () => postAiBatch(runId, customer, plan.market, batch),+ )),+ );+ } catch (error) {+ // Its answers fail at finalize.+ console.warn(`[ai-visibility] ${runId} repost-${name}:`, error);+ }+ }+ };try {plan = await pgStep(step, "prepare", SINGLE_ATTEMPT, () =>prepareAiRun(runId, customer),@@ -121,18 +155,7 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<// Its unsaved answers fail at finalize; the other batches continue.console.warn(`[ai-visibility] ${runId} model-${index}:`, error);}- if (!pending.length) continue;- const remaining = pending;- try {- pending = await pgStep(- step,- `collect-model-${index}`,- COLLECT_STEP,- () => collectAiRound(runId, remaining),- );- } catch (error) {- console.warn(`[ai-visibility] ${runId} collect-model-${index}:`, error);- }+ if (pending.length) await collect(`model-${index}`, true);}for (let round = 0;@@ -148,15 +171,8 @@ export class AiVisibilityWorkflow extends WorkflowEntrypoint<// log it as one.return markFailed(error);}- const remaining = pending;- try {- pending = await pgStep(step, `collect-${round}`, COLLECT_STEP, () =>- collectAiRound(runId, remaining),- );- } catch (error) {- // Posted tasks are already paid for; keep polling them next round.- console.warn(`[ai-visibility] ${runId} collect-${round}:`, error);- }+ // A task posted after the last round would never be read.+ await collect(`${round}`, round < POLL_INTERVALS.length - 1);}try {await pgStep(step, "finalize", SINGLE_ATTEMPT, () =>