Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
import { z } from "zod";
import type {
BrowserRun,
Response as BrowserResponse,
} from "@cloudflare/workers-types";
import {
CLOUDFLARE_RENDER_TIMEOUT_MS,
type RenderUsage,
} from "@/shared/audit-rendering";
import { classifyFetch } from "./classify-fetch";
import { MAX_HTML_BYTES, readTextUpTo, truncateToBytes } from "./html-response";
import { isCrawlableUrl } from "./url-policy";
import { normalizeUrl } from "./url-utils";
type RenderOptions = {
// Either renderer may be missing: Docker has no browser binding, and the
// Context key is optional on self-hosted Cloudflare deployments.
browser?: Pick<BrowserRun, "quickAction">;
// Reaches Context.dev authenticated (provider-access.ts).
contextFetch?: typeof fetch | null;
/** Logged with each attempt; reads outside an audit have none. */
auditId?: string;
// Counts every provider attempt, including rejected ones: both providers
// bill a request whether or not the HTML turned out to be usable.
usage: RenderUsage;
};
const cloudflareSchema = z.object({
success: z.literal(true),
result: z.string(),
meta: z.object({
status: z.number().int(),
finalUrl: z.string().url(),
headers: z.record(z.string(), z.string()).optional(),
}),
});
// Context includes key_metadata on every authenticated response, error or not.
const usageSchema = z.object({
key_metadata: z
.object({ credits_consumed: z.number().nonnegative().optional() })
.optional(),
});
const contextSchema = z.object({
success: z.literal(true),
type: z.literal("html"),
html: z.string(),
metadata: z.object({ finalUrl: z.string().url() }),
finalDOMState: z.literal("loaded"),
});
// JSON escaping can expand an HTML byte up to sixfold. Bound the transport
// before JSON.parse as well as the decoded HTML before it reaches the parser.
// A body cut off at the bound is incomplete JSON, so parsing rejects it.
const MAX_RESPONSE_BYTES = 8 * MAX_HTML_BYTES;
async function readProviderJson(
response: Response | BrowserResponse,
): Promise<unknown> {
return JSON.parse(await readTextUpTo(response, MAX_RESPONSE_BYTES));
}
/**
* The rendered HTML to analyze: the same first 1 MiB the direct crawl keeps,
* which still holds the metadata and navigation the audit reads.
*/
function usableHtml(url: string, finalUrl: string, html: string) {
// The existing direct fetch records redirects one hop at a time. A browser
// must not silently substitute another URL's content into this page row.
if (normalizeUrl(finalUrl, url) !== normalizeUrl(url, url)) {
throw new Error("Rendering returned a different URL");
}
if (!html.trim()) throw new Error("Rendered HTML is empty");
return truncateToBytes(html, MAX_HTML_BYTES);
}
/** The rendered HTML, with the origin's status when the renderer reports it. */
export type RenderedPage = { html: string; status: number | null };
/** Null when Cloudflare failed, was blocked, or saw an origin error. */
async function renderWithCloudflare(
url: string,
browser: Pick<BrowserRun, "quickAction">,
): Promise<RenderedPage | null> {
try {
const response = await browser.quickAction("content", {
url,
cacheTTL: 0,
gotoOptions: {
waitUntil: "networkidle2",
timeout: CLOUDFLARE_RENDER_TIMEOUT_MS,
},
});
if (!response.ok) {
await response.body?.cancel();
return null;
}
const data = cloudflareSchema.parse(await readProviderJson(response));
const { status, finalUrl, headers } = data.meta;
const html = usableHtml(url, finalUrl, data.result);
const mitigated = Object.entries(headers ?? {}).some(
([name, value]) =>
name.toLowerCase() === "cf-mitigated" && Boolean(value.trim()),
);
if (
status >= 400 ||
classifyFetch(status, mitigated, html.slice(0, 4_000)) !== "ok"
) {
return null;
}
return { html, status };
} catch {
// Provider bodies/errors can contain page data. Nothing raw is logged.
return null;
}
}
async function renderWithContext(
url: string,
contextFetch: typeof fetch,
): Promise<{
html: string | null;
credits: number;
httpStatus: number | null;
}> {
const endpoint = new URL("https://api.context.dev/v1/web/scrape/html");
endpoint.search = new URLSearchParams({
url,
maxAgeMs: "0",
useMainContentOnly: "false",
waitForMs: "1000",
"pdf[shouldParse]": "false",
"timeoutOpts[milliseconds]": "30000",
"timeoutOpts[behavior]": "fail",
}).toString();
let html: string | null = null;
let creditsUsed: number | null = null;
let httpStatus: number | null = null;
try {
const response = await contextFetch(endpoint, {
signal: AbortSignal.timeout(35_000),
});
httpStatus = response.status;
const json = await readProviderJson(response);
// Usage first: a rejected or partial document still cost a credit.
const usage = usageSchema.safeParse(json);
creditsUsed = usage.success
? (usage.data.key_metadata?.credits_consumed ?? null)
: null;
if (!response.ok) throw new Error(`Rendering provider HTTP ${httpStatus}`);
const data = contextSchema.parse(json);
html = usableHtml(url, data.metadata.finalUrl, data.html);
} catch {
// Provider bodies/errors can contain page data or credentials. Nothing
// raw is logged, and a paid call is never retried automatically.
}
// Context charges one credit per scrape. Bill what it reports; only assume
// one credit for a usable page that omits its usage, never for a failure.
return { html, credits: creditsUsed ?? (html ? 1 : 0), httpStatus };
}
/**
* Cloudflare Browser Run first, because it costs a fraction of Context. Context
* renders only the pages Cloudflare could not: a failure, a bot challenge, or
* an origin error.
*/
async function renderPage(
url: string,
options: RenderOptions,
): Promise<RenderedPage> {
if (!isCrawlableUrl(url) || new URL(url).username || new URL(url).password) {
throw new Error("Invalid rendering target");
}
if (options.browser) {
const page = await renderWithCloudflare(url, options.browser);
options.usage.cloudflareAttempts += 1;
console.info("site_audit:render", {
auditId: options.auditId,
provider: "cloudflare",
outcome: page ? "ok" : "fallback",
});
if (page) return page;
}
if (options.contextFetch) {
const { html, credits, httpStatus } = await renderWithContext(
url,
options.contextFetch,
);
options.usage.contextCredits += credits;
console.info("site_audit:render", {
auditId: options.auditId,
provider: "context",
httpStatus,
creditsUsed: credits,
outcome: html ? "ok" : "error",
});
// Context reports no origin status.
if (html) return { html, status: null };
}
throw new Error("Unable to render this page");
}
export const RenderedPageService = { renderPage } as const;