Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
import { env } from "cloudflare:workers";
import {
customerHasPaidPlan,
type BillingCustomerContext,
} from "@/server/billing/subscription";
import {
createDataforseoClient,
fetchKeywordMetricsForList,
} from "@/server/lib/dataforseo";
import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
import { assertSerpLocationNameAccepted } from "@/server/lib/dataforseo/serp-location-validate";
import { AppError } from "@/server/lib/errors";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
import type {
RankCheckScheduleTime,
RankTrackingConfig,
RankCheckTriggerResult,
} from "@/types/schemas/rank-tracking";
import {
beginRankCheckRun,
reconcileActiveRankCheckRun,
} from "./rankCheckRunGuards";
import {
estimateRankCheckCredits,
MAX_CONFIGS_PER_PROJECT,
rankCheckCostApprovalError,
} from "@/shared/rank-tracking";
import { resolveNextCheckAt } from "./configSchedule";
import {
getIsoCountryCode,
resolveKeywordDataLanguage,
resolveMarket,
} from "@/shared/keyword-locations";
import { getLatestResults } from "./rankTrackingResults";
import { toRankTrackingTimestamp } from "@/server/features/rank-tracking/rankTrackingTimestamps";
import { RankTrackingKeywordService } from "./RankTrackingKeywordService";
// ---------------------------------------------------------------------------
// Config
// ---------------------------------------------------------------------------
async function createConfig(input: {
projectId: string;
projectMarket: { locationCode: number; languageCode: string };
domain: string;
locationCode?: number;
languageCode?: string;
locationName?: string;
devices?: RankTrackingConfig["devices"];
serpDepth: number;
scheduleInterval?: RankTrackingConfig["scheduleInterval"];
scheduleTime?: RankCheckScheduleTime;
}) {
const normalizedDomain = normalizeDomain(input.domain);
const { locationCode, languageCode } = resolveMarket(
input,
input.projectMarket,
);
const scheduleInterval = input.scheduleInterval ?? "weekly";
const nextCheckAt = resolveNextCheckAt(scheduleInterval, input.scheduleTime);
const locationName = input.locationName ?? null;
// Before the duplicate/limit checks so an unusable location name is the
// error the caller sees.
if (locationName) {
await assertSerpLocationNameAccepted({
locationName,
languageCode,
countryCode: getIsoCountryCode(locationCode),
});
}
const existing = await RankTrackingRepository.getConfigByProjectDomainMarket(
input.projectId,
normalizedDomain,
locationCode,
languageCode,
locationName,
);
// The (project, domain, location, language) row still exists when a domain is
// archived — archiving only flips isActive to false. So re-adding an
// archived domain reactivates that row (keeping its keyword/ranking
// history) with the freshly chosen settings, rather than colliding with
// the unique index. An already-active row is a genuine duplicate.
if (existing?.isActive) {
throw new AppError(
"VALIDATION_ERROR",
locationName
? "This domain + city + language combination is already being tracked"
: "This domain + country + language combination is already being tracked",
);
}
// Enforced for reactivations too, not just new rows — otherwise archiving
// and re-adding domains would push a project past the active-config cap.
const allConfigs = await RankTrackingRepository.getConfigsForProject(
input.projectId,
);
if (allConfigs.length >= MAX_CONFIGS_PER_PROJECT) {
throw new AppError(
"VALIDATION_ERROR",
`Maximum ${MAX_CONFIGS_PER_PROJECT} tracked domains per project`,
);
}
if (existing) {
await RankTrackingRepository.updateConfig(existing.id, input.projectId, {
isActive: true,
devices: input.devices ?? "both",
serpDepth: input.serpDepth,
scheduleInterval,
nextCheckAt,
// Drop any stale skip reason from before it was archived so the
// re-added domain doesn't surface an outdated warning.
lastSkipReason: null,
});
return getValidatedConfig(existing.id, input.projectId);
}
const configId = crypto.randomUUID();
const config: RankTrackingConfig = {
id: configId,
projectId: input.projectId,
domain: normalizedDomain,
locationCode,
languageCode,
locationName,
devices: input.devices ?? "both",
serpDepth: input.serpDepth,
scheduleInterval,
nextCheckAt,
isActive: true,
lastCheckedAt: null,
lastSkipReason: null,
createdAt: toRankTrackingTimestamp(new Date()),
};
await RankTrackingRepository.createConfig(config);
return config;
}
async function updateConfig(
configId: string,
projectId: string,
input: {
domain?: string;
locationCode?: number;
languageCode?: string;
locationName?: string | null;
devices?: RankTrackingConfig["devices"];
serpDepth?: number;
scheduleInterval?: RankTrackingConfig["scheduleInterval"];
scheduleTime?: RankCheckScheduleTime;
isActive?: boolean;
},
) {
const { scheduleTime, ...fields } = input;
const updates: typeof fields & { nextCheckAt?: string | null } = {};
const existing = await getValidatedConfig(configId, projectId);
// A location name is only valid together with its market, so re-check the
// resulting (name, language, country) whenever any of the three changes.
const marketChanged =
input.locationName !== undefined ||
input.locationCode !== undefined ||
input.languageCode !== undefined;
if (marketChanged) {
const locationName =
input.locationName === undefined
? existing.locationName
: input.locationName;
if (locationName) {
await assertSerpLocationNameAccepted({
locationName,
languageCode: input.languageCode ?? existing.languageCode,
countryCode: getIsoCountryCode(
input.locationCode ?? existing.locationCode,
),
});
}
}
if (input.domain !== undefined)
updates.domain = normalizeDomain(input.domain);
if (input.locationCode !== undefined)
updates.locationCode = input.locationCode;
if (input.languageCode !== undefined)
updates.languageCode = input.languageCode;
if (input.locationName !== undefined)
updates.locationName = input.locationName;
if (input.devices !== undefined) updates.devices = input.devices;
if (input.serpDepth !== undefined) updates.serpDepth = input.serpDepth;
if (input.isActive !== undefined) updates.isActive = input.isActive;
const scheduleInterval = input.scheduleInterval ?? existing.scheduleInterval;
// The config modal resends the interval on every save, so only move the
// anchor when the schedule really changed — otherwise editing, say, devices
// would re-randomize the run time.
if (
scheduleTime ||
scheduleInterval !== existing.scheduleInterval ||
(scheduleInterval !== "manual" && !existing.nextCheckAt)
) {
updates.scheduleInterval = scheduleInterval;
updates.nextCheckAt = resolveNextCheckAt(scheduleInterval, scheduleTime);
}
await RankTrackingRepository.updateConfig(configId, projectId, updates);
}
// ---------------------------------------------------------------------------
// Trigger a manual check
// ---------------------------------------------------------------------------
async function triggerCheck(input: {
configId: string;
projectId: string;
billingCustomer: BillingCustomerContext;
keywordIds?: string[];
maxCostCredits?: number;
}): Promise<RankCheckTriggerResult> {
const config = await getValidatedConfig(input.configId, input.projectId);
await requireRankCheckAccess(input.billingCustomer.organizationId);
const keywords = await RankTrackingRepository.getKeywordsForConfig(config.id);
if (keywords.length === 0) {
throw new AppError(
"INTERNAL_ERROR",
"No keywords to track. Add keywords to this domain first.",
);
}
if (input.maxCostCredits != null) {
const { costCredits } = estimateRankCheckCredits(
keywords.map((kw) => kw.keyword),
config.devices,
config.serpDepth,
"live",
);
if (costCredits > input.maxCostCredits) {
throw new AppError(
"VALIDATION_ERROR",
rankCheckCostApprovalError(costCredits, input.maxCostCredits),
);
}
}
return beginRankCheckRun({
workflow: env.RANK_CHECK_WORKFLOW,
config,
projectId: input.projectId,
billingCustomer: {
userId: input.billingCustomer.userId,
userEmail: input.billingCustomer.userEmail,
organizationId: input.billingCustomer.organizationId,
projectId: input.billingCustomer.projectId,
},
keywordsTotal: input.keywordIds ? input.keywordIds.length : keywords.length,
keywordIds: input.keywordIds,
maxCostCredits: input.maxCostCredits,
trigger: "manual",
workflowStartErrorMessage: "Failed to start rank check workflow",
});
}
// A scheduled check this close covers the same keywords, so an automatic
// check now would bill the customer twice within the hour.
const AUTO_CHECK_SCHEDULE_WINDOW_MS = 60 * 60_000;
// The check that follows adding a domain or keywords, as opposed to the user
// pressing "Check Now" — so it yields to an imminent scheduled check.
async function triggerAutoCheck(input: {
configId: string;
projectId: string;
billingCustomer: BillingCustomerContext;
keywordIds?: string[];
}): Promise<
| RankCheckTriggerResult
| { ok: false; reason: "scheduled_soon" | "no_keywords" }
> {
const config = await getValidatedConfig(input.configId, input.projectId);
const msUntilScheduled = config.nextCheckAt
? new Date(config.nextCheckAt).getTime() - Date.now()
: Infinity;
if (msUntilScheduled <= AUTO_CHECK_SCHEDULE_WINDOW_MS) {
// A free plan must not be told to wait for a scheduled check that the
// scheduler will skip. The other path checks access in triggerCheck.
await requireRankCheckAccess(input.billingCustomer.organizationId);
return { ok: false, reason: "scheduled_soon" };
}
// A brand-new domain has no keywords yet; a re-added archived one keeps its.
const counts = await RankTrackingRepository.getKeywordCountsForConfigs([
config.id,
]);
if (!counts.get(config.id)) return { ok: false, reason: "no_keywords" };
return triggerCheck(input);
}
async function getLatestRun(configId: string, projectId: string) {
await getValidatedConfig(configId, projectId);
const run = await RankTrackingRepository.getLatestRunForConfig(configId);
if (!run) return null;
// If the DB says the run is still active, check the workflow instance.
// We only report staleness here — the next call to beginRankCheckRun will
// mark a stale blocker as failed before retrying its insert. Mutating from
// this read path caused a race where the original workflow kept running
// while a replacement was started.
const reconciliation = await reconcileActiveRankCheckRun(run);
if (reconciliation) {
return formatRun(run, {
maybeStale: true,
staleReason: reconciliation.errorMessage,
});
}
return formatRun(run);
}
// ---------------------------------------------------------------------------
// Keyword metrics (volume, difficulty, CPC)
// ---------------------------------------------------------------------------
async function refreshKeywordMetrics(
configId: string,
projectId: string,
billingCustomer: BillingCustomerContext,
): Promise<{ updated: number }> {
const config = await getValidatedConfig(configId, projectId);
await requireRankCheckAccess(billingCustomer.organizationId);
const keywords = await RankTrackingRepository.getKeywordsForConfig(configId);
if (keywords.length === 0) return { updated: 0 };
const client = createDataforseoClient(billingCustomer);
const metrics = await fetchKeywordMetricsForList(client, {
// The keyword-data APIs are case-insensitive and echo keywords back
// lowercased, so ask in lowercase. A match-case keyword can sit next to
// its lowercase twin; both then map to the same metrics row and the
// request carries no duplicates.
keywords: [...new Set(keywords.map((kw) => kw.keyword.toLowerCase()))],
locationCode: config.locationCode,
// Trackers can pair any SERP language with any country; the keyword-data
// APIs only serve the country's own languages.
languageCode: resolveKeywordDataLanguage(
config.locationCode,
config.languageCode,
),
// Local configs get volume/CPC scoped to the tracked city; national
// numbers can overstate local demand by orders of magnitude.
locationName: config.locationName ?? undefined,
creditFeature: "rank_tracking",
});
const byKeyword = new Map(
metrics.map((metric) => [metric.keyword.toLowerCase(), metric]),
);
const now = new Date().toISOString();
const updates = keywords
.map((kw) => {
const metric = byKeyword.get(kw.keyword.toLowerCase());
if (!metric) return null;
// Rank tracking only tracks volume / difficulty / CPC.
return {
id: kw.id,
searchVolume: metric.searchVolume,
keywordDifficulty: metric.keywordDifficulty,
cpc: metric.cpc,
metricsFetchedAt: now,
};
})
.filter((u): u is NonNullable<typeof u> => u !== null);
if (updates.length === 0) return { updated: 0 };
await RankTrackingRepository.updateKeywordMetrics(updates);
return { updated: updates.length };
}
// ---------------------------------------------------------------------------
// MCP/browser read models and access policy
// ---------------------------------------------------------------------------
async function getConfigs(projectId: string) {
return RankTrackingRepository.getConfigsForProject(projectId);
}
async function getTracker(configId: string, projectId: string) {
const config = await getValidatedConfig(configId, projectId);
const results = await getLatestResults(configId, projectId);
return { config, results };
}
async function requireRankCheckAccess(organizationId: string) {
if (!(await isHostedServerAuthMode())) return;
if (await customerHasPaidPlan(organizationId)) return;
throw new AppError(
"PAYMENT_REQUIRED",
"Upgrade to the paid plan to run rank checks",
);
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
async function getValidatedConfig(configId: string, projectId: string) {
const config = await RankTrackingRepository.getConfigById({
configId,
projectId,
});
if (!config) {
throw new AppError("NOT_FOUND", "Rank tracking config not found");
}
return config;
}
function normalizeDomain(domain: string): string {
let d = domain.trim().toLowerCase();
// Strip protocol
d = d.replace(/^https?:\/\//, "");
// Strip path, query string, and fragment
d = d.replace(/[/?#].*$/, "");
// Strip trailing slash
d = d.replace(/\/+$/, "");
// Strip www. prefix
d = d.replace(/^www\./, "");
if (!d) {
throw new AppError("INTERNAL_ERROR", "Invalid domain");
}
return d;
}
type RunRow = NonNullable<
Awaited<ReturnType<typeof RankTrackingRepository.getLatestRunForConfig>>
>;
function formatRun(
run: RunRow,
stale?: { maybeStale: boolean; staleReason: string },
) {
return {
id: run.id,
status: run.status,
keywordsTotal: run.keywordsTotal,
keywordsChecked: run.keywordsChecked,
isSubsetRun: run.isSubsetRun,
errorMessage: run.errorMessage,
startedAt: run.startedAt,
completedAt: run.completedAt,
maybeStale: stale?.maybeStale ?? false,
staleReason: stale?.staleReason ?? null,
};
}
export const RankTrackingService = {
createConfig,
updateConfig,
addKeywords: RankTrackingKeywordService.addKeywords,
removeKeywords: RankTrackingKeywordService.removeKeywords,
triggerCheck,
triggerAutoCheck,
getLatestRun,
estimateCost: RankTrackingKeywordService.estimateCost,
refreshKeywordMetrics,
getConfigs,
getTracker,
requireRankCheckAccess,
};