Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Read any page setup can't fetch through Context, not just throttled Shopify pages (#978)
Carter Smith965a9c19002e
diff --git a/src/server/features/projects/services/websiteResearch.ts b/src/server/features/projects/services/websiteResearch.tsindex 99fcf362..a8c33c56 100644--- a/src/server/features/projects/services/websiteResearch.ts+++ b/src/server/features/projects/services/websiteResearch.ts@@ -110,7 +110,7 @@ export async function researchWebsite(// Each field is independent: saved context supplies the evidence, and the// model only receives output fields and tools for the missing information.const spend: ResearchSpend[] = [];- // Throttled Shopify pages are read through Context, billed with the research.+ // Pages the plain fetch can't read go through Context, billed with the research.const contextUsage: RenderUsage = {cloudflareAttempts: 0,contextCredits: 0,diff --git a/src/server/lib/scrape.test.ts b/src/server/lib/scrape.test.tsindex 1c33cfba..8836d657 100644--- a/src/server/lib/scrape.test.ts+++ b/src/server/lib/scrape.test.ts@@ -104,21 +104,18 @@ describe("readSite", () => {});});-describe("readSite on a throttled Shopify store", () => {- afterEach(() => {- vi.unstubAllGlobals();- });-- it("reads the page through Context and counts its credit", async () => {+describe("readSite with Context usage", () => {+ // The homepage is blocked, /internal redirects to a private host, and other+ // URLs are missing; Context renders pages.+ beforeEach(() => {vi.stubGlobal("fetch",vi.fn(async (input: string | URL) => {const url = String(input);if (url.includes("dns-query"))- // In Shopify's range, so the 429 below is Shopify's edge.return Response.json({Status: 0,- Answer: [{ type: 1, data: "23.227.38.65" }],+ Answer: [{ type: 1, data: "93.184.216.34" }],});if (url.startsWith("https://api.context.dev/"))return Response.json({@@ -129,9 +126,23 @@ describe("readSite on a throttled Shopify store", () => {finalDOMState: "loaded",key_metadata: { credits_consumed: 1 },});- return new Response(null, { status: 429 });+ if (url === "https://store.example.com/")+ return new Response(null, { status: 403 });+ if (url === "https://store.example.com/internal")+ return new Response(null, {+ status: 302,+ headers: { location: "http://169.254.169.254/" },+ });+ return new Response(null, { status: 404 });}),);+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("reads a blocked page through Context and counts its credit", async () => {const usage = { cloudflareAttempts: 0, contextCredits: 0 };const result = await readSite("store.example.com", 3, usage);@@ -145,4 +156,20 @@ describe("readSite on a throttled Shopify store", () => {]);expect(usage.contextCredits).toBe(1);});++ it("does not spend Context credits on a missing page", async () => {+ const usage = { cloudflareAttempts: 0, contextCredits: 0 };++ await readPages(["https://store.example.com/missing"], 1, usage);++ expect(usage.contextCredits).toBe(0);+ });++ it("does not send a redirect to a blocked host to Context", async () => {+ const usage = { cloudflareAttempts: 0, contextCredits: 0 };++ await readPages(["https://store.example.com/internal"], 1, usage);++ expect(usage.contextCredits).toBe(0);+ });});diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.tsindex 972cfffa..0062f90f 100644--- a/src/server/lib/scrape.ts+++ b/src/server/lib/scrape.ts@@ -5,10 +5,7 @@// gracefully (less text); a Browser Rendering upgrade can slot in behind this// same interface later.-import {- isShopifyHost,- normalizeAndValidateStartUrl,-} from "@/server/lib/audit/url-policy";+import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";import { RenderedPageService } from "@/server/lib/audit/rendered-page";import { getOptionalEnvValue } from "@/server/lib/runtime-env";import type { RenderUsage } from "@/shared/audit-rendering";@@ -58,31 +55,17 @@ export async function readBoundedText(}/**- * Shopify's edge throttles requests from Cloudflare Workers, so a throttled- * storefront page is read through Context instead. Its credits are added to- * `usage` for the caller to bill.+ * With `contextUsage`, a page the plain fetch can't read is read through+ * Context, because many sites throttle or block Cloudflare Workers (Shopify's+ * edge, bot protection). Its credits are added to `contextUsage` for the caller+ * to bill. A missing page (404/410) is not retried there.*/-async function readThrottledShopifyPage(- url: string,- usage: RenderUsage,-): Promise<string | null> {- const contextApiKey = await getOptionalEnvValue("CONTEXT_API_KEY");- if (!contextApiKey || !(await isShopifyHost(new URL(url).hostname)))- return null;- try {- return (await RenderedPageService.renderPage(url, { contextApiKey, usage }))- .html;- } catch {- return null;- }-}-async function fetchText(url: string,contextUsage?: RenderUsage,): Promise<string | null> {+ let current = url;try {- let current = url;for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {const response = await fetch(current, {headers: { "user-agent": USER_AGENT, accept: "text/html,*/*" },@@ -91,21 +74,37 @@ async function fetchText(redirect: "manual",signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),});- if (response.status === 429 && contextUsage) {+ if (response.ok) return await readBoundedText(response);+ const location = response.headers.get("location");+ if (response.status < 300 || response.status >= 400 || !location) {await response.body?.cancel();- return readThrottledShopifyPage(current, contextUsage);+ if (response.status === 404 || response.status === 410) return null;+ break;}- if (response.status < 300 || response.status >= 400)- return response.ok ? await readBoundedText(response) : null;- const location = response.headers.get("location");- if (!location) return null;// Re-validates host, blocks private IPs, and does DoH DNS resolution.- // Throws on a blocked or invalid destination.- current = await normalizeAndValidateStartUrl(- new URL(location, current).toString(),- );+ // A blocked or invalid destination is not sent to Context either: it+ // follows redirects itself and would reach the blocked host.+ try {+ current = await normalizeAndValidateStartUrl(+ new URL(location, current).toString(),+ );+ } catch {+ return null;+ }}- return null;+ } catch {+ // Timeouts, TLS and network errors fall through to Context.+ }+ if (!contextUsage) return null;+ const contextApiKey = await getOptionalEnvValue("CONTEXT_API_KEY");+ if (!contextApiKey) return null;+ try {+ return (+ await RenderedPageService.renderPage(current, {+ contextApiKey,+ usage: contextUsage,+ })+ ).html;} catch {return null;}@@ -188,7 +187,7 @@ async function scrapePage(* exact pages (the user's own or a competitor's) rather than asking us to* discover a site. Each URL is independently run through the SSRF guard, so a* blocked or unreachable URL is skipped rather than failing the batch.- * With `contextUsage`, throttled Shopify pages are read through Context.+ * With `contextUsage`, pages the plain fetch can't read are read through Context.*/export async function readPages(urls: string[],