Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Retry failed manual rank checks through the task queue (EVE-411) (#905)
Carter Smith9a1afcbf4733
diff --git a/src/server/workflows/RankCheckWorkflow.ts b/src/server/workflows/RankCheckWorkflow.tsindex 1a60e43f..af4532b5 100644--- a/src/server/workflows/RankCheckWorkflow.ts+++ b/src/server/workflows/RankCheckWorkflow.ts@@ -377,12 +377,12 @@ export class RankCheckWorkflow extends WorkflowEntrypoint<runId,};// Scheduled checks use DataForSEO's task queue (~30% of live cost);- // manual checks stay on the live endpoint for instant results.- if (trigger === "scheduled") {- queueStats = await runQueuedCheck(step, checkContext);- } else {- await runLiveCheck(step, checkContext);- }+ // manual checks stay on the live endpoint for instant results, and+ // only the pairs the live endpoint missed go to the queue.+ queueStats =+ trigger === "scheduled"+ ? await runQueuedCheck(step, checkContext)+ : await runLiveCheck(step, checkContext);} catch (error) {// Batch failure — snapshots for completed batches are already// persisted incrementally. Continue to finalization.diff --git a/src/server/workflows/rankCheckPaths.test.ts b/src/server/workflows/rankCheckPaths.test.tsnew file mode 100644index 00000000..9fc771b0--- /dev/null+++ b/src/server/workflows/rankCheckPaths.test.ts@@ -0,0 +1,125 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { WorkflowStep } from "cloudflare:workers";+import type { RankCheckTaskInput } from "@/server/lib/dataforseo";+import { runLiveCheck } from "./rankCheckPaths";++const mocks = vi.hoisted(() => ({+ getSnapshotsForRun: vi.fn(),+ insertSnapshots: vi.fn(),+ updateRun: vi.fn(),+ fetchRankCheckTaskResult: vi.fn(),+ rankCheckBatch: vi.fn(),+ rankCheckTaskPost: vi.fn(),+}));++vi.mock(+ "@/server/features/rank-tracking/repositories/RankTrackingRepository",+ () => ({ RankTrackingRepository: mocks }),+);+vi.mock("@/server/lib/dataforseo", () => ({+ fetchRankCheckTaskResult: mocks.fetchRankCheckTaskResult,+ MAX_TASKS_PER_POST: 100,+}));+vi.mock("@/server/workflows/pgStep", () => ({+ pgStep: (+ _step: unknown,+ _name: string,+ _config: unknown,+ fn: () => unknown,+ ) => fn(),+}));++const keywords = Array.from({ length: 11 }, (_, index) => ({+ id: `kw_${index}`,+ keyword: `keyword ${index}`,+}));++function runManualCheck() {+ return runLiveCheck(+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only sleep is reached; pgStep is mocked+ { sleep: vi.fn() } as unknown as WorkflowStep,+ {+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the check paths only call these two serp methods+ client: {+ serp: {+ rankCheckBatch: mocks.rankCheckBatch,+ rankCheckTaskPost: mocks.rankCheckTaskPost,+ },+ } as never,+ keywords,+ devices: "desktop",+ serpDepth: 10,+ domain: "openseo.so",+ locationCode: 2840,+ languageCode: "en",+ runId: "run_1",+ },+ );+}++describe("runLiveCheck", () => {+ beforeEach(() => {+ mocks.rankCheckBatch.mockImplementation(+ async (inputs: RankCheckTaskInput[]) =>+ inputs.map((input) => ({+ status: "fulfilled",+ value: { ...input, position: 3, url: null, serpFeatures: [] },+ })),+ );+ mocks.rankCheckTaskPost.mockImplementation(+ async ({ tasks }: { tasks: RankCheckTaskInput[] }) =>+ tasks.map((task) => ({ ...task, taskId: `task_${task.keywordId}` })),+ );+ mocks.fetchRankCheckTaskResult.mockImplementation(+ async (task: RankCheckTaskInput) => ({+ status: "completed",+ result: { ...task, position: 5, url: null, serpFeatures: [] },+ }),+ );+ });++ it("keeps going after a failed live batch and queues only the missing pairs", async () => {+ // The first batch step dies after its calls; the second batch saves kw_10.+ mocks.insertSnapshots.mockRejectedValueOnce(new Error("step timed out"));+ mocks.getSnapshotsForRun.mockResolvedValue([+ { trackingKeywordId: "kw_10", device: "desktop" },+ ]);++ const stats = await runManualCheck();++ expect(mocks.rankCheckBatch).toHaveBeenCalledTimes(2);+ expect(mocks.rankCheckTaskPost).toHaveBeenCalledOnce();+ expect(mocks.rankCheckTaskPost).toHaveBeenCalledWith(+ expect.objectContaining({+ tasks: keywords+ .slice(0, 10)+ .map((kw): unknown => expect.objectContaining({ keywordId: kw.id })),+ }),+ );+ expect(stats).toMatchObject({ queueTasks: 10, queueCollected: 10 });+ });++ it("makes no second live call when a queued retry fails", async () => {+ mocks.getSnapshotsForRun.mockResolvedValue([]);+ mocks.fetchRankCheckTaskResult.mockResolvedValue({+ status: "failed",+ message: "Internal SE Server Error.",+ });++ const stats = await runManualCheck();++ // Two live batches, then no live fallback for the failed queue tasks:+ // that call could exceed the credits the user approved.+ expect(mocks.rankCheckBatch).toHaveBeenCalledTimes(2);+ expect(stats).toMatchObject({ queueTasks: 11, fallbackTasks: 0 });+ });++ it("skips the queue when the live endpoint checked every pair", async () => {+ mocks.getSnapshotsForRun.mockResolvedValue(+ keywords.map((kw) => ({ trackingKeywordId: kw.id, device: "desktop" })),+ );++ expect(await runManualCheck()).toBeNull();+ expect(mocks.rankCheckTaskPost).not.toHaveBeenCalled();+ });+});diff --git a/src/server/workflows/rankCheckPaths.ts b/src/server/workflows/rankCheckPaths.tsindex 994ac15d..c5b76bc3 100644--- a/src/server/workflows/rankCheckPaths.ts+++ b/src/server/workflows/rankCheckPaths.ts@@ -141,31 +141,83 @@ async function checkBatchLive(* Snapshots are written incrementally after each batch so partial results* survive batch failures. ~6s per keyword batch.* Each batch is billed with one credit hold and one settle (checkBatchLive).+ * A failed batch step doesn't stop the run. Afterwards, every keyword/device+ * pair still without a snapshot goes through the task queue (runQueuedCheck),+ * which retries timeouts and in-band task errors at queue price. Failed live+ * calls aren't billed, so the retry stays within the live estimate the user+ * approved. Exception: a step that died after its calls were metered gets+ * billed again at queue price. Returns the queue stats, or null when nothing+ * was missing.*/export async function runLiveCheck(step: WorkflowStep,ctx: CheckContext,-): Promise<void> {+): Promise<QueuedCheckStats | null> {for (let i = 0; i < ctx.keywords.length; i += KEYWORDS_PER_BATCH) {const keywordBatch = ctx.keywords.slice(i, i + KEYWORDS_PER_BATCH);const batchTasks = expandToTaskInputs(keywordBatch, ctx.devices);const batchIndex = Math.floor(i / KEYWORDS_PER_BATCH);const keywordsChecked = i + keywordBatch.length;- await pgStep(- step,- `live-batch-${batchIndex}`,- SINGLE_ATTEMPT_STEP_CONFIG,- async () => {- const written = await checkBatchLive(ctx, batchTasks);- // Progress for the UI; finalize recounts from the DB anyway.- await RankTrackingRepository.updateRun(ctx.runId, {- keywordsChecked,- });- return written;- },- );+ try {+ await pgStep(+ step,+ `live-batch-${batchIndex}`,+ SINGLE_ATTEMPT_STEP_CONFIG,+ async () => {+ const written = await checkBatchLive(ctx, batchTasks);+ // Progress for the UI; finalize recounts from the DB anyway.+ await RankTrackingRepository.updateRun(ctx.runId, {+ keywordsChecked,+ });+ return written;+ },+ );+ } catch (error) {+ // Earlier batches are already saved; this batch's pairs go to the+ // queue below.+ console.warn(+ `[rank-check] ${ctx.runId} live-batch-${batchIndex} failed:`,+ error,+ );+ }}++ // Diff against the DB rather than batch return values, so pairs from a+ // batch step that threw are included too.+ const missing = await pgStep(+ step,+ "find-missing",+ COLLECT_STEP_CONFIG,+ async () => {+ const snapshots = await RankTrackingRepository.getSnapshotsForRun(+ ctx.runId,+ );+ const checked = new Set(+ snapshots.map((s) => `${s.trackingKeywordId}:${s.device}`),+ );+ // Live progress counts attempted keywords; reset it to the real count+ // while the queue runs.+ await RankTrackingRepository.updateRun(ctx.runId, {+ keywordsChecked: new Set(snapshots.map((s) => s.trackingKeywordId))+ .size,+ });+ return expandToTaskInputs(ctx.keywords, ctx.devices).filter(+ (t) => !checked.has(`${t.keywordId}:${t.device}`),+ );+ },+ );+ if (missing.length === 0) return null;++ console.log(+ `[rank-check] ${ctx.runId} queue fallback for ${missing.length} task(s)`,+ );+ // No live fallback here: these pairs already had their live attempt, and a+ // second one could take the run past the credits the user approved.+ return runQueuedCheck(step, ctx, {+ taskInputs: missing,+ liveFallback: false,+ });}// Poll cadence for queued tasks. Standard-priority tasks complete in ~5@@ -245,8 +297,8 @@ async function collectQueuedRound(console.warn(`[rank-check] ${ctx.runId} task ${task.taskId} failed: ${result.value.message}`,);- // No run error here: every failed task gets a live retry, which- // records its own error if it fails too.+ // No run error here: a live call records it. That is the live+ // fallback, or for a manual retry, the live attempt before the queue.failed.push(task);} else {completed.push({ ...result.value.result, device: task.device });@@ -284,18 +336,21 @@ export interface QueuedCheckStats {/*** Check keywords via DataForSEO's standard task queue (~30% of live cost).- * Posts every keyword/device pair as a queued task, then polls task_get for- * ~15 minutes, writing snapshots incrementally as tasks complete. Anything- * still unfinished after the polling window — plus tasks DataForSEO rejected- * or failed — gets one shot at the live endpoint so a run never hangs on a- * stuck queue. Billing happens at task_post (and per live-fallback batch).+ * Posts every keyword/device pair (or just taskInputs) as a queued task, then+ * polls task_get for ~15 minutes, writing snapshots incrementally as tasks+ * complete. Anything still unfinished after the polling window — plus tasks+ * DataForSEO rejected or failed — gets one shot at the live endpoint so a run+ * never hangs on a stuck queue, unless `liveFallback` is false. Billing+ * happens at task_post (and per live-fallback batch).*/export async function runQueuedCheck(step: WorkflowStep,ctx: CheckContext,+ {+ taskInputs = expandToTaskInputs(ctx.keywords, ctx.devices),+ liveFallback = true,+ }: { taskInputs?: RankCheckTaskInput[]; liveFallback?: boolean } = {},): Promise<QueuedCheckStats> {- const taskInputs = expandToTaskInputs(ctx.keywords, ctx.devices);-// Post all tasks to the queue, <=100 per request, one metered step each.// A failed chunk must not abort the run — earlier chunks were already// charged at DataForSEO, so their results have to be collected. The failed@@ -387,8 +442,14 @@ export async function runQueuedCheck(// Progress isn't updated here; finalize recounts keywordsChecked from the// DB.const stragglers: RankCheckTaskInput[] = [...fallback, ...pending];- stats.fallbackTasks = stragglers.length;if (stragglers.length === 0) return stats;+ if (!liveFallback) {+ console.log(+ `[rank-check] ${ctx.runId} ${stragglers.length} task(s) left unchecked, no live fallback`,+ );+ return stats;+ }+ stats.fallbackTasks = stragglers.length;console.log(`[rank-check] ${ctx.runId} live fallback for ${stragglers.length} task(s)`,