Skip to content

Open source alternative to Semrush and Ahrefs

Retry failed manual rank checks through the task queue (EVE-411) (#905)

Carter Smith9a1afcbf4733
diff --git a/src/server/workflows/RankCheckWorkflow.ts b/src/server/workflows/RankCheckWorkflow.ts
index 1a60e43f..af4532b5 100644
--- a/src/server/workflows/RankCheckWorkflow.ts
+++ b/src/server/workflows/RankCheckWorkflow.ts
@@ -377,12 +377,12 @@ export class RankCheckWorkflow extends WorkflowEntrypoint<
runId,
};
// Scheduled checks use DataForSEO's task queue (~30% of live cost);
- // manual checks stay on the live endpoint for instant results.
- if (trigger === "scheduled") {
- queueStats = await runQueuedCheck(step, checkContext);
- } else {
- await runLiveCheck(step, checkContext);
- }
+ // manual checks stay on the live endpoint for instant results, and
+ // only the pairs the live endpoint missed go to the queue.
+ queueStats =
+ trigger === "scheduled"
+ ? await runQueuedCheck(step, checkContext)
+ : await runLiveCheck(step, checkContext);
} catch (error) {
// Batch failure — snapshots for completed batches are already
// persisted incrementally. Continue to finalization.
diff --git a/src/server/workflows/rankCheckPaths.test.ts b/src/server/workflows/rankCheckPaths.test.ts
new file mode 100644
index 00000000..9fc771b0
--- /dev/null
+++ b/src/server/workflows/rankCheckPaths.test.ts
@@ -0,0 +1,125 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { WorkflowStep } from "cloudflare:workers";
+import type { RankCheckTaskInput } from "@/server/lib/dataforseo";
+import { runLiveCheck } from "./rankCheckPaths";
+
+const mocks = vi.hoisted(() => ({
+ getSnapshotsForRun: vi.fn(),
+ insertSnapshots: vi.fn(),
+ updateRun: vi.fn(),
+ fetchRankCheckTaskResult: vi.fn(),
+ rankCheckBatch: vi.fn(),
+ rankCheckTaskPost: vi.fn(),
+}));
+
+vi.mock(
+ "@/server/features/rank-tracking/repositories/RankTrackingRepository",
+ () => ({ RankTrackingRepository: mocks }),
+);
+vi.mock("@/server/lib/dataforseo", () => ({
+ fetchRankCheckTaskResult: mocks.fetchRankCheckTaskResult,
+ MAX_TASKS_PER_POST: 100,
+}));
+vi.mock("@/server/workflows/pgStep", () => ({
+ pgStep: (
+ _step: unknown,
+ _name: string,
+ _config: unknown,
+ fn: () => unknown,
+ ) => fn(),
+}));
+
+const keywords = Array.from({ length: 11 }, (_, index) => ({
+ id: `kw_${index}`,
+ keyword: `keyword ${index}`,
+}));
+
+function runManualCheck() {
+ return runLiveCheck(
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only sleep is reached; pgStep is mocked
+ { sleep: vi.fn() } as unknown as WorkflowStep,
+ {
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the check paths only call these two serp methods
+ client: {
+ serp: {
+ rankCheckBatch: mocks.rankCheckBatch,
+ rankCheckTaskPost: mocks.rankCheckTaskPost,
+ },
+ } as never,
+ keywords,
+ devices: "desktop",
+ serpDepth: 10,
+ domain: "openseo.so",
+ locationCode: 2840,
+ languageCode: "en",
+ runId: "run_1",
+ },
+ );
+}
+
+describe("runLiveCheck", () => {
+ beforeEach(() => {
+ mocks.rankCheckBatch.mockImplementation(
+ async (inputs: RankCheckTaskInput[]) =>
+ inputs.map((input) => ({
+ status: "fulfilled",
+ value: { ...input, position: 3, url: null, serpFeatures: [] },
+ })),
+ );
+ mocks.rankCheckTaskPost.mockImplementation(
+ async ({ tasks }: { tasks: RankCheckTaskInput[] }) =>
+ tasks.map((task) => ({ ...task, taskId: `task_${task.keywordId}` })),
+ );
+ mocks.fetchRankCheckTaskResult.mockImplementation(
+ async (task: RankCheckTaskInput) => ({
+ status: "completed",
+ result: { ...task, position: 5, url: null, serpFeatures: [] },
+ }),
+ );
+ });
+
+ it("keeps going after a failed live batch and queues only the missing pairs", async () => {
+ // The first batch step dies after its calls; the second batch saves kw_10.
+ mocks.insertSnapshots.mockRejectedValueOnce(new Error("step timed out"));
+ mocks.getSnapshotsForRun.mockResolvedValue([
+ { trackingKeywordId: "kw_10", device: "desktop" },
+ ]);
+
+ const stats = await runManualCheck();
+
+ expect(mocks.rankCheckBatch).toHaveBeenCalledTimes(2);
+ expect(mocks.rankCheckTaskPost).toHaveBeenCalledOnce();
+ expect(mocks.rankCheckTaskPost).toHaveBeenCalledWith(
+ expect.objectContaining({
+ tasks: keywords
+ .slice(0, 10)
+ .map((kw): unknown => expect.objectContaining({ keywordId: kw.id })),
+ }),
+ );
+ expect(stats).toMatchObject({ queueTasks: 10, queueCollected: 10 });
+ });
+
+ it("makes no second live call when a queued retry fails", async () => {
+ mocks.getSnapshotsForRun.mockResolvedValue([]);
+ mocks.fetchRankCheckTaskResult.mockResolvedValue({
+ status: "failed",
+ message: "Internal SE Server Error.",
+ });
+
+ const stats = await runManualCheck();
+
+ // Two live batches, then no live fallback for the failed queue tasks:
+ // that call could exceed the credits the user approved.
+ expect(mocks.rankCheckBatch).toHaveBeenCalledTimes(2);
+ expect(stats).toMatchObject({ queueTasks: 11, fallbackTasks: 0 });
+ });
+
+ it("skips the queue when the live endpoint checked every pair", async () => {
+ mocks.getSnapshotsForRun.mockResolvedValue(
+ keywords.map((kw) => ({ trackingKeywordId: kw.id, device: "desktop" })),
+ );
+
+ expect(await runManualCheck()).toBeNull();
+ expect(mocks.rankCheckTaskPost).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/server/workflows/rankCheckPaths.ts b/src/server/workflows/rankCheckPaths.ts
index 994ac15d..c5b76bc3 100644
--- a/src/server/workflows/rankCheckPaths.ts
+++ b/src/server/workflows/rankCheckPaths.ts
@@ -141,31 +141,83 @@ async function checkBatchLive(
* Snapshots are written incrementally after each batch so partial results
* survive batch failures. ~6s per keyword batch.
* Each batch is billed with one credit hold and one settle (checkBatchLive).
+ * A failed batch step doesn't stop the run. Afterwards, every keyword/device
+ * pair still without a snapshot goes through the task queue (runQueuedCheck),
+ * which retries timeouts and in-band task errors at queue price. Failed live
+ * calls aren't billed, so the retry stays within the live estimate the user
+ * approved. Exception: a step that died after its calls were metered gets
+ * billed again at queue price. Returns the queue stats, or null when nothing
+ * was missing.
*/
export async function runLiveCheck(
step: WorkflowStep,
ctx: CheckContext,
-): Promise<void> {
+): Promise<QueuedCheckStats | null> {
for (let i = 0; i < ctx.keywords.length; i += KEYWORDS_PER_BATCH) {
const keywordBatch = ctx.keywords.slice(i, i + KEYWORDS_PER_BATCH);
const batchTasks = expandToTaskInputs(keywordBatch, ctx.devices);
const batchIndex = Math.floor(i / KEYWORDS_PER_BATCH);
const keywordsChecked = i + keywordBatch.length;
- await pgStep(
- step,
- `live-batch-${batchIndex}`,
- SINGLE_ATTEMPT_STEP_CONFIG,
- async () => {
- const written = await checkBatchLive(ctx, batchTasks);
- // Progress for the UI; finalize recounts from the DB anyway.
- await RankTrackingRepository.updateRun(ctx.runId, {
- keywordsChecked,
- });
- return written;
- },
- );
+ try {
+ await pgStep(
+ step,
+ `live-batch-${batchIndex}`,
+ SINGLE_ATTEMPT_STEP_CONFIG,
+ async () => {
+ const written = await checkBatchLive(ctx, batchTasks);
+ // Progress for the UI; finalize recounts from the DB anyway.
+ await RankTrackingRepository.updateRun(ctx.runId, {
+ keywordsChecked,
+ });
+ return written;
+ },
+ );
+ } catch (error) {
+ // Earlier batches are already saved; this batch's pairs go to the
+ // queue below.
+ console.warn(
+ `[rank-check] ${ctx.runId} live-batch-${batchIndex} failed:`,
+ error,
+ );
+ }
}
+
+ // Diff against the DB rather than batch return values, so pairs from a
+ // batch step that threw are included too.
+ const missing = await pgStep(
+ step,
+ "find-missing",
+ COLLECT_STEP_CONFIG,
+ async () => {
+ const snapshots = await RankTrackingRepository.getSnapshotsForRun(
+ ctx.runId,
+ );
+ const checked = new Set(
+ snapshots.map((s) => `${s.trackingKeywordId}:${s.device}`),
+ );
+ // Live progress counts attempted keywords; reset it to the real count
+ // while the queue runs.
+ await RankTrackingRepository.updateRun(ctx.runId, {
+ keywordsChecked: new Set(snapshots.map((s) => s.trackingKeywordId))
+ .size,
+ });
+ return expandToTaskInputs(ctx.keywords, ctx.devices).filter(
+ (t) => !checked.has(`${t.keywordId}:${t.device}`),
+ );
+ },
+ );
+ if (missing.length === 0) return null;
+
+ console.log(
+ `[rank-check] ${ctx.runId} queue fallback for ${missing.length} task(s)`,
+ );
+ // No live fallback here: these pairs already had their live attempt, and a
+ // second one could take the run past the credits the user approved.
+ return runQueuedCheck(step, ctx, {
+ taskInputs: missing,
+ liveFallback: false,
+ });
}
// Poll cadence for queued tasks. Standard-priority tasks complete in ~5
@@ -245,8 +297,8 @@ async function collectQueuedRound(
console.warn(
`[rank-check] ${ctx.runId} task ${task.taskId} failed: ${result.value.message}`,
);
- // No run error here: every failed task gets a live retry, which
- // records its own error if it fails too.
+ // No run error here: a live call records it. That is the live
+ // fallback, or for a manual retry, the live attempt before the queue.
failed.push(task);
} else {
completed.push({ ...result.value.result, device: task.device });
@@ -284,18 +336,21 @@ export interface QueuedCheckStats {
/**
* Check keywords via DataForSEO's standard task queue (~30% of live cost).
- * Posts every keyword/device pair as a queued task, then polls task_get for
- * ~15 minutes, writing snapshots incrementally as tasks complete. Anything
- * still unfinished after the polling window — plus tasks DataForSEO rejected
- * or failed — gets one shot at the live endpoint so a run never hangs on a
- * stuck queue. Billing happens at task_post (and per live-fallback batch).
+ * Posts every keyword/device pair (or just taskInputs) as a queued task, then
+ * polls task_get for ~15 minutes, writing snapshots incrementally as tasks
+ * complete. Anything still unfinished after the polling window — plus tasks
+ * DataForSEO rejected or failed — gets one shot at the live endpoint so a run
+ * never hangs on a stuck queue, unless `liveFallback` is false. Billing
+ * happens at task_post (and per live-fallback batch).
*/
export async function runQueuedCheck(
step: WorkflowStep,
ctx: CheckContext,
+ {
+ taskInputs = expandToTaskInputs(ctx.keywords, ctx.devices),
+ liveFallback = true,
+ }: { taskInputs?: RankCheckTaskInput[]; liveFallback?: boolean } = {},
): Promise<QueuedCheckStats> {
- const taskInputs = expandToTaskInputs(ctx.keywords, ctx.devices);
-
// Post all tasks to the queue, <=100 per request, one metered step each.
// A failed chunk must not abort the run — earlier chunks were already
// charged at DataForSEO, so their results have to be collected. The failed
@@ -387,8 +442,14 @@ export async function runQueuedCheck(
// Progress isn't updated here; finalize recounts keywordsChecked from the
// DB.
const stragglers: RankCheckTaskInput[] = [...fallback, ...pending];
- stats.fallbackTasks = stragglers.length;
if (stragglers.length === 0) return stats;
+ if (!liveFallback) {
+ console.log(
+ `[rank-check] ${ctx.runId} ${stragglers.length} task(s) left unchecked, no live fallback`,
+ );
+ return stats;
+ }
+ stats.fallbackTasks = stragglers.length;
console.log(
`[rank-check] ${ctx.runId} live fallback for ${stragglers.length} task(s)`,