Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
feat(rank-tracking): raise scheduler throughput 5x with rate-limit-derived sizing (#465)
Ben Senescueec998a76233
diff --git a/src/server.ts b/src/server.tsindex 9d476f33..5132a938 100644--- a/src/server.ts+++ b/src/server.ts@@ -188,15 +188,20 @@ export default {env: Env,_ctx: ExecutionContext,) {- // Scope a per-request Postgres client for the cron run (no-op in D1 mode).- // Caught so a rank-tracking failure can't suppress the audit watchdog below.+ // Watchdog first: reconcile audits stuck in "running" whose workflow died+ // without reaching mark-failed (OOM/CPU kills, expired instances). Runs+ // before the rank loop so a slow tick can't delay or starve it. Its+ // failure is held until after the rank checks so it can't suppress them,+ // then rethrown so the invocation still reports as failed.+ let watchdogError: unknown;try {- await withPgClient(() => runScheduledRankChecks(env));+ await withPgClient(() => reconcileStaleAudits());} catch (err) {- console.error("[cron] Scheduled rank checks failed:", err);+ watchdogError = err;+ console.error("[cron] Stale-audit reconcile failed:", err);}- // Watchdog: reconcile audits stuck in "running" whose workflow died- // without reaching mark-failed (OOM/CPU kills, expired instances).- await withPgClient(() => reconcileStaleAudits());+ // Scope a per-request Postgres client for the cron run (no-op in D1 mode).+ await withPgClient(() => runScheduledRankChecks(env));+ if (watchdogError) throw watchdogError;},};diff --git a/src/server/features/rank-tracking/repositories/RankTrackingRepository.ts b/src/server/features/rank-tracking/repositories/RankTrackingRepository.tsindex 0cc8933f..0fb49d80 100644--- a/src/server/features/rank-tracking/repositories/RankTrackingRepository.ts+++ b/src/server/features/rank-tracking/repositories/RankTrackingRepository.ts@@ -115,6 +115,11 @@ async function updateConfig();}+// Caps per-tick loop work (claims, per-org plan checks) against the cron+// wall clock; paid-heavy ticks are stopped earlier by the unit budget and+// slow ticks by TICK_DEADLINE_MS in scheduledRankChecks.ts.+const DUE_CONFIGS_PER_TICK = 500;+async function getDueConfigsWithOrganization(nowIso: string) {return (db@@ -150,7 +155,7 @@ async function getDueConfigsWithOrganization(nowIso: string) {asc(rankTrackingConfigs.nextCheckAt),asc(rankTrackingConfigs.id),)- .limit(200)+ .limit(DUE_CONFIGS_PER_TICK));}diff --git a/src/server/features/rank-tracking/services/scheduledRankChecks.test.ts b/src/server/features/rank-tracking/services/scheduledRankChecks.test.tsindex 94611e76..345e1ab4 100644--- a/src/server/features/rank-tracking/services/scheduledRankChecks.test.ts+++ b/src/server/features/rank-tracking/services/scheduledRankChecks.test.ts@@ -1,4 +1,5 @@import { beforeEach, describe, expect, it, vi } from "vitest";+import { MAX_KEYWORDS_PER_CONFIG } from "@/shared/rank-tracking";type DueConfigRow = {id: string;@@ -177,15 +178,16 @@ describe("runScheduledRankChecks", () => {});it("stops admitting configs once the task-unit budget is spent", async () => {- // 400 keywords × 2 devices = 800 units — admitted anyway as the tick's- // first start (oversized exemption), then exhausts the 200-unit budget.+ // A legal-max config (MAX_KEYWORDS_PER_CONFIG × both devices) exceeds the+ // whole budget but is admitted as the tick's first start (oversized+ // exemption); the next config then hits the budget stop.mocks.getDueConfigsWithOrganization.mockResolvedValue([dueConfig({ id: "config_big" }),dueConfig({ id: "config_next", nextCheckAt: "2026-01-02T00:00:00.000Z" }),]);mocks.getKeywordCountsForConfigs.mockResolvedValue(new Map([- ["config_big", 400],+ ["config_big", MAX_KEYWORDS_PER_CONFIG],["config_next", 5],]),);@@ -198,7 +200,10 @@ describe("runScheduledRankChecks", () => {expect(mocks.claimDueConfig).toHaveBeenCalledTimes(1);expect(mocks.customerHasPaidPlan).toHaveBeenCalledTimes(1);expect(logSpy).toHaveBeenCalledWith(- expect.objectContaining({ stoppedByBudget: true, unitsStarted: 800 }),+ expect.objectContaining({+ stoppedByBudget: true,+ unitsStarted: MAX_KEYWORDS_PER_CONFIG * 2,+ }),);});@@ -252,8 +257,9 @@ describe("runScheduledRankChecks", () => {});it("defers a config whose units would overflow the remaining budget", async () => {- // First config fits (10 units); the second would overflow (800), so it is- // deferred rather than admitted just because budget remains.+ // First config fits (10 units); the second (legal-max, over the whole+ // budget) would overflow, so it is deferred rather than admitted just+ // because budget remains.mocks.getDueConfigsWithOrganization.mockResolvedValue([dueConfig({ id: "config_small" }),dueConfig({ id: "config_big", nextCheckAt: "2026-01-02T00:00:00.000Z" }),@@ -261,7 +267,7 @@ describe("runScheduledRankChecks", () => {mocks.getKeywordCountsForConfigs.mockResolvedValue(new Map([["config_small", 5],- ["config_big", 400],+ ["config_big", MAX_KEYWORDS_PER_CONFIG],]),);const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});@@ -277,6 +283,26 @@ describe("runScheduledRankChecks", () => {);});+ it("stops the tick at the wall-clock deadline instead of running long", async () => {+ mocks.getDueConfigsWithOrganization.mockResolvedValue([+ dueConfig({ id: "config_1" }),+ dueConfig({ id: "config_2", nextCheckAt: "2026-01-02T00:00:00.000Z" }),+ ]);+ const start = Date.now();+ vi.spyOn(Date, "now")+ .mockReturnValueOnce(start) // deadline anchor+ .mockReturnValue(start + 4 * 60_000); // every later check is past it+ const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});++ await runTick();++ expect(mocks.beginRankCheckRun).not.toHaveBeenCalled();+ expect(mocks.claimDueConfig).not.toHaveBeenCalled();+ expect(logSpy).toHaveBeenCalledWith(+ expect.objectContaining({ stoppedByDeadline: true, started: 0 }),+ );+ });+it("restores the original due time when a run is already active", async () => {mocks.beginRankCheckRun.mockResolvedValue({ok: false,diff --git a/src/server/features/rank-tracking/services/scheduledRankChecks.ts b/src/server/features/rank-tracking/services/scheduledRankChecks.tsindex 7f6f6be8..c24cd578 100644--- a/src/server/features/rank-tracking/services/scheduledRankChecks.ts+++ b/src/server/features/rank-tracking/services/scheduledRankChecks.ts@@ -10,9 +10,24 @@ import {// Work admitted per tick, in task units (keywords × devices). Admission// control, not a hard rate limit: the first start of a tick is always-// admitted, so an oversized config can never starve. 200 units on the-// 5-minute cron ≈ 57,600 units/day, ~9× current steady-state demand.-const SCHEDULED_TASK_UNIT_BUDGET = 200;+// admitted, so a config bigger than the budget (legal max: 1,000 keywords ×+// 2 devices = 2,000 units) can never starve. Sized against DataForSEO's+// 2,000 requests/min account cap, where task_get polling is the binding+// term: one call per unit per poll round, rounds wake synchronized per tick,+// and up to three ticks' ~15-minute poll windows overlap the */5 cron — so a+// full tick can burst ~1,000 polls into a single minute, stacking with the+// residual rounds of the two prior ticks. Overruns aren't+// loud failures: throttled polls age into the live fallback at ~3× cost,+// billed to the customer, so keep real headroom under the cap.+// 1,000/tick ≈ 288,000 units/day, ~45× steady-state demand — it only binds+// during backlog catch-up.+const SCHEDULED_TASK_UNIT_BUDGET = 1000;++// Wall-clock guard for the per-config loop: sub-hourly crons are killed at 15+// minutes, and a skip-heavy tick pays serial Autumn round-trips per distinct+// org (worst case minutes, more when Autumn is degraded). Stopping early is+// safe — unprocessed configs stay due and the next tick resumes oldest-first.+const TICK_DEADLINE_MS = 3 * 60_000;// Cap on the per-tick list of configs blocked by an active run. Blocked// configs leave no durable trace on their row, so the summary names them.@@ -43,9 +58,11 @@ export async function runScheduledRankChecks(env: Env) {return check;};+ const deadline = Date.now() + TICK_DEADLINE_MS;let unitsStarted = 0;let started = 0;let stoppedByBudget = false;+ let stoppedByDeadline = false;let skippedFree = 0;let skippedNoKeywords = 0;let concurrentChangeSkips = 0;@@ -56,6 +73,10 @@ export async function runScheduledRankChecks(env: Env) {let configErrors = 0;for (const config of dueConfigs) {+ if (Date.now() >= deadline) {+ stoppedByDeadline = true;+ break;+ }// Per-config containment: one bad row (e.g. malformed next_check_at, which// sorts first and would head every scan) or transient DB error must not// starve the rest of the tick or suppress the summary log.@@ -219,6 +240,7 @@ export async function runScheduledRankChecks(env: Env) {unitsStarted,budget: SCHEDULED_TASK_UNIT_BUDGET,stoppedByBudget,+ stoppedByDeadline,skippedFree,skippedNoKeywords,concurrentChangeSkips,diff --git a/src/server/workflows/rankCheckPaths.ts b/src/server/workflows/rankCheckPaths.tsindex f77971d4..949ce394 100644--- a/src/server/workflows/rankCheckPaths.ts+++ b/src/server/workflows/rankCheckPaths.ts@@ -166,7 +166,7 @@ const QUEUED_POLL_INTERVALS = [/** Concurrent task_get requests within a collect step. */const TASK_GET_CONCURRENCY = 25;-/** Max task_get calls per collect round (per-invocation subrequest budget). */+/** Max task_get calls per collect round (bounds one round's fan-out). */const TASK_GETS_PER_COLLECT = 500;// Collect steps may issue hundreds of task_get calls, so they get more room