Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
import { env, type WorkflowStep } from "cloudflare:workers";
import { NonRetryableError } from "cloudflare:workflows";
import type { RobotsResult } from "@/server/lib/audit/discovery";
import type { AuditConfig, CrawledPageResult } from "@/server/lib/audit/types";
import {
getAuditScratchpad,
type ClaimedUrl,
} from "@/server/features/audit/AuditScratchpad";
import { persistCrawledPages } from "@/server/workflows/siteAuditWorkflowPersist";
import {
adjustCrawlWindow,
clampCrawlWindow,
CRAWL_WINDOW,
RETRY_CRAWL_WINDOW,
} from "@/server/lib/audit/crawl-window";
import {
createCrawlThrottle,
type CrawlThrottleState,
} from "@/server/lib/audit/crawl-throttle";
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
import {
estimateRenderingCredits,
renderUsageCredits,
type RenderUsage,
} from "@/shared/audit-rendering";
import type { CrawlerAccess } from "@/shared/crawler-access";
import { pgStep } from "@/server/workflows/pgStep";
import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
import { getContextFetch } from "@/server/lib/provider-access";
/**
* The crawl runs in chunks: each chunk is one durable step that leases up to
* CHUNK_TARGET_PAGES URLs from the scratchpad DO, crawls them with a rolling
* concurrency window, and persists results incrementally (full page rows to
* the app DB, links/mirror/frontier updates to the DO). Step returns carry
* only counters, so step state stays tiny regardless of site size and the
* workflow heap stays O(one batch).
*/
const CHUNK_TARGET_PAGES = 200;
/** Stop launching new fetches after this long; leftover leases are released. */
const CHUNK_SOFT_DEADLINE_MS = 90_000;
/** Crawled pages are persisted in sub-batches of this size. */
const PERSIST_BATCH_SIZE = 25;
/**
* The first sub-batch of a chunk is deliberately small: the crawl window
* only adapts when a sub-batch persists, and on a heavy-page site a full
* 25-page batch crawled at the starting window was already enough to
* exceed the isolate's memory. Five pages tell the byte bound what the
* site's pages weigh before the window commits to more.
*/
const FIRST_PERSIST_BATCH_SIZE = 5;
/**
* Stop launching new fetches while more than this many persist sub-batches
* are waiting: persistence is sequential, so when the DB falls behind a fast
* site, unpersisted page results would otherwise pile up in memory without
* bound.
*/
const MAX_QUEUED_PERSIST_BATCHES = 2;
type CrawlPhaseParams = {
auditId: string;
workflowInstanceId: string;
origin: string;
maxPages: number;
robots: RobotsResult;
/** Frontier size after discovery seeding (from the discover-urls step). */
seededCount: number;
renderJavaScript?: boolean;
sitePlatform?: AuditConfig["sitePlatform"];
/**
* Rendering totals across chunks. Mutated as each chunk step returns, so a
* failed audit can still settle what its completed chunks rendered.
*/
renderUsage: RenderUsage;
/** Crawler-access headers for the audited host, when the org has one. */
access?: CrawlerAccess | null;
/**
* A metered rendered audit's credit hold. No fetch starts after its
* deadline or once rendering it could cost more than it holds.
*/
renderHold?: { deadlineAt: number; credits: number };
};
export type CrawlPhaseResult = {
pagesCrawled: number;
/** True when the frontier was exhausted before hitting maxPages. */
completed: boolean;
/** Why the crawl stopped before running out of URLs or pages. */
stoppedBy?:
| "crawl-rate-limited"
| "crawl-time-limit"
| "crawl-credits-exhausted";
};
export async function runCrawlPhase(
step: WorkflowStep,
params: CrawlPhaseParams,
): Promise<CrawlPhaseResult> {
let chunkNo = 0;
let attemptedTotal = 0;
let pending = params.seededCount;
let zeroProgressChunks = 0;
// The adapted window carries across chunks via durable step results:
// without this every chunk restarted at the initial window and re-learned
// the site's page weight the hard way — on heavy-page sites that meant an
// exceededMemory death every ~200 pages.
let windowHint = CRAWL_WINDOW.initial;
let throttleState: CrawlThrottleState | undefined;
while (pending > 0 && attemptedTotal < params.maxPages) {
chunkNo += 1;
const result = await pgStep(
step,
`crawl-chunk-${chunkNo}`,
CRAWL_CHUNK_STEP,
() =>
runCrawlChunk({
...params,
chunkNo,
attemptedBefore: attemptedTotal,
startWindow: windowHint,
throttleState,
}),
);
// Apply the chunk's counters even when it did no new work (a retried
// chunk whose earlier attempt persisted everything reports 0 attempts
// with up-to-date scratchpad totals) — finalize must not see stale ones.
attemptedTotal = result.attempted;
pending = result.pending;
params.renderUsage.cloudflareAttempts +=
result.renderUsage?.cloudflareAttempts ?? 0;
params.renderUsage.contextCredits +=
result.renderUsage?.contextCredits ?? 0;
const stoppedBy = result.rateLimited
? "crawl-rate-limited"
: result.timedOut
? "crawl-time-limit"
: result.outOfCredits && "crawl-credits-exhausted";
if (stoppedBy) {
return { pagesCrawled: attemptedTotal, completed: false, stoppedBy };
}
// `?? initial`: an instance in flight across a deploy replays cached
// step results from before endWindow existed.
windowHint = result.endWindow ?? CRAWL_WINDOW.initial;
throttleState = result.throttleState;
if (pending > 0 && attemptedTotal < params.maxPages && result.resumeAt) {
// The timestamp comes from the persisted chunk result. Always replay
// the same sleep step, even when that timestamp is now in the past.
await step.sleepUntil(`crawl-cooldown-${chunkNo}`, result.resumeAt);
zeroProgressChunks = 0;
continue;
}
// One zero-attempt chunk is normal (retry of a completed chunk number);
// two in a row means the frontier is unservable — stop with what we
// have instead of spinning forever.
zeroProgressChunks =
result.attemptedInChunk === 0 ? zeroProgressChunks + 1 : 0;
if (zeroProgressChunks >= 2) break;
}
return { pagesCrawled: attemptedTotal, completed: pending === 0 };
}
async function runCrawlChunk(
input: CrawlPhaseParams & {
chunkNo: number;
attemptedBefore: number;
startWindow: number;
throttleState?: CrawlThrottleState;
},
): Promise<{
attemptedInChunk: number;
attempted: number;
pending: number;
endWindow: number;
rateLimited?: boolean;
timedOut?: boolean;
outOfCredits?: boolean;
throttleState?: CrawlThrottleState;
resumeAt?: number;
renderUsage?: RenderUsage;
}> {
const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
input;
const scratchpad = getAuditScratchpad(auditId);
const renderDeadlineAt = input.renderHold?.deadlineAt ?? Infinity;
let previousThrottle =
(await scratchpad.getCrawlThrottle()) ?? input.throttleState;
const claimLimit = Math.min(
CHUNK_TARGET_PAGES,
maxPages - input.attemptedBefore,
);
const { urls: claimed, isRetry } = await scratchpad.claimChunk(
chunkNo,
claimLimit,
);
const deadlineAt = Math.min(
Date.now() + CHUNK_SOFT_DEADLINE_MS,
renderDeadlineAt,
);
if (isRetry && previousThrottle) {
// Requests made since the last checkpoint may have consumed a slot.
previousThrottle = {
...previousThrottle,
nextRequestAt: Math.max(
previousThrottle.nextRequestAt,
Date.now() + previousThrottle.intervalMs,
),
};
}
const throttle = createCrawlThrottle(
deadlineAt,
previousThrottle,
async (state) => {
try {
await scratchpad.saveCrawlThrottle(state);
} catch {
// A retry cannot safely honor a cooldown that failed to checkpoint.
throw new NonRetryableError(
"Unable to save the site's crawl cooldown.",
);
}
},
);
if (claimed.length === 0) {
const stats = await scratchpad.getStats();
return {
attemptedInChunk: 0,
attempted: stats.attempted,
pending: stats.pending,
endWindow: input.startWindow,
throttleState: throttle.state,
rateLimited: throttle.stopped,
resumeAt:
throttle.state.pausedUntil > Date.now()
? throttle.state.pausedUntil
: undefined,
};
}
const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));
// A retry means the previous attempt died mid-crawl (in production almost
// always exceededMemory), and it is the chunk's last attempt — so it runs
// under drastically reduced limits instead of the profile that just failed.
const limits = isRetry ? RETRY_CRAWL_WINDOW : CRAWL_WINDOW;
let windowSize = isRetry
? limits.initial
: clampCrawlWindow(input.startWindow, limits);
let nextIndex = 0;
let attemptedInChunk = 0;
const inFlight = new Set<Promise<void>>();
const deferred: string[] = [];
let persistThreshold = FIRST_PERSIST_BATCH_SIZE;
let batch: CrawledPageResult[] = [];
// Persistence runs concurrently with fetching (pipelined) but sequentially
// with itself, so DB write pressure stays bounded at one batch at a time.
let persistChain: Promise<unknown> = Promise.resolve();
let queuedPersists = 0;
const flush = () => {
if (batch.length === 0) return;
const pages = batch;
batch = [];
persistThreshold = PERSIST_BATCH_SIZE;
windowSize = adjustCrawlWindow(windowSize, pages, limits);
queuedPersists += 1;
persistChain = persistChain
.then(() =>
persistCrawledPages({
auditId,
workflowInstanceId,
origin,
robots,
sitePlatform: input.sitePlatform,
scratchpad,
pages,
depthByUrl,
maxPages,
}),
)
.finally(() => {
queuedPersists -= 1;
});
};
// This attempt's rendering counts, returned in the step result. The audit's
// credit lock is settled once, when the audit ends.
const renderUsage: RenderUsage = { cloudflareAttempts: 0, contextCredits: 0 };
// Whether `pending` in-flight pages and one more, each at the Context worst
// case, still fit the hold on top of what this audit has rendered so far.
const canAffordRender = (pending: number) =>
!input.renderHold ||
renderUsageCredits({
cloudflareAttempts:
input.renderUsage.cloudflareAttempts + renderUsage.cloudflareAttempts,
contextCredits:
input.renderUsage.contextCredits + renderUsage.contextCredits,
}) +
estimateRenderingCredits(pending + 1).high <=
input.renderHold.credits;
const render = input.renderJavaScript
? async (url: string) => {
const { RenderedPageService } =
await import("@/server/lib/audit/rendered-page");
return RenderedPageService.renderPage(url, {
browser: env.BROWSER,
// Behind an Every App gateway its calls are the audit's
// organization's, named around the Workflow's run() and carried
// into this step by pgStep.
contextFetch: await getContextFetch(),
auditId,
usage: renderUsage,
});
}
: undefined;
const launch = (entry: ClaimedUrl) => {
const promise = crawlPage(
entry.url,
entry.depth,
entry.inSitemap,
throttle,
{
access: input.access,
render,
renderThrottled: input.sitePlatform === "shopify",
},
)
.then((page) => {
if (!page) {
deferred.push(entry.url);
return;
}
attemptedInChunk += 1;
batch.push(page);
if (batch.length >= persistThreshold) flush();
})
.finally(() => {
inFlight.delete(promise);
});
inFlight.add(promise);
};
while (true) {
while (
inFlight.size < windowSize &&
nextIndex < claimed.length &&
Date.now() < deadlineAt
) {
// queuedPersists changes when persistChain settles. Keep it out of the
// loop condition because the type-aware linter cannot see that async
// mutation and flags the otherwise valid backpressure check.
if (
throttle.stopped ||
queuedPersists > MAX_QUEUED_PERSIST_BATCHES ||
!canAffordRender(inFlight.size)
)
break;
launch(claimed[nextIndex]);
nextIndex += 1;
}
if (inFlight.size > 0) {
await Promise.race(inFlight);
continue;
}
// Nothing in flight. If launches are only paused by persistence
// backpressure, wait for the queue to drain and resume; otherwise the
// chunk is done (leases exhausted or soft deadline hit).
if (
!throttle.stopped &&
queuedPersists > MAX_QUEUED_PERSIST_BATCHES &&
nextIndex < claimed.length &&
Date.now() < deadlineAt
) {
await persistChain;
continue;
}
break;
}
flush();
await persistChain;
await scratchpad.saveCrawlThrottle(throttle.state);
// Preserve URLs without a page result, including slots stopped by a cooldown.
const unattempted = [
...deferred,
...claimed.slice(nextIndex).map((entry) => entry.url),
];
if (unattempted.length > 0) {
await scratchpad.releaseUrls(unattempted);
}
// Progress counters are written per persisted sub-batch (in
// persistCrawledPages), so a chunk that dies mid-way underreports by at
// most one sub-batch, not a whole chunk.
const stats = await scratchpad.getStats();
const throttleState = throttle.state;
return {
attemptedInChunk,
attempted: stats.attempted,
pending: stats.pending,
endWindow: windowSize,
rateLimited: throttle.stopped,
outOfCredits: unattempted.length > 0 && !canAffordRender(0),
// Work is left and neither now nor the site's pacing allows another
// request before the deadline, so no later chunk could start one either.
timedOut:
unattempted.length > 0 &&
Math.max(
Date.now(),
throttleState.pausedUntil,
throttleState.nextRequestAt,
) >= renderDeadlineAt,
throttleState,
resumeAt:
throttleState.pausedUntil > Date.now()
? throttleState.pausedUntil
: undefined,
renderUsage,
};
}