Skip to content

Open source alternative to Semrush and Ahrefs

Read any page setup can't fetch through Context, not just throttled Shopify pages (#978)

Carter Smith965a9c19002e
diff --git a/src/server/features/projects/services/websiteResearch.ts b/src/server/features/projects/services/websiteResearch.ts
index 99fcf362..a8c33c56 100644
--- a/src/server/features/projects/services/websiteResearch.ts
+++ b/src/server/features/projects/services/websiteResearch.ts
@@ -110,7 +110,7 @@ export async function researchWebsite(
// Each field is independent: saved context supplies the evidence, and the
// model only receives output fields and tools for the missing information.
const spend: ResearchSpend[] = [];
- // Throttled Shopify pages are read through Context, billed with the research.
+ // Pages the plain fetch can't read go through Context, billed with the research.
const contextUsage: RenderUsage = {
cloudflareAttempts: 0,
contextCredits: 0,
diff --git a/src/server/lib/scrape.test.ts b/src/server/lib/scrape.test.ts
index 1c33cfba..8836d657 100644
--- a/src/server/lib/scrape.test.ts
+++ b/src/server/lib/scrape.test.ts
@@ -104,21 +104,18 @@ describe("readSite", () => {
});
});
-describe("readSite on a throttled Shopify store", () => {
- afterEach(() => {
- vi.unstubAllGlobals();
- });
-
- it("reads the page through Context and counts its credit", async () => {
+describe("readSite with Context usage", () => {
+ // The homepage is blocked, /internal redirects to a private host, and other
+ // URLs are missing; Context renders pages.
+ beforeEach(() => {
vi.stubGlobal(
"fetch",
vi.fn(async (input: string | URL) => {
const url = String(input);
if (url.includes("dns-query"))
- // In Shopify's range, so the 429 below is Shopify's edge.
return Response.json({
Status: 0,
- Answer: [{ type: 1, data: "23.227.38.65" }],
+ Answer: [{ type: 1, data: "93.184.216.34" }],
});
if (url.startsWith("https://api.context.dev/"))
return Response.json({
@@ -129,9 +126,23 @@ describe("readSite on a throttled Shopify store", () => {
finalDOMState: "loaded",
key_metadata: { credits_consumed: 1 },
});
- return new Response(null, { status: 429 });
+ if (url === "https://store.example.com/")
+ return new Response(null, { status: 403 });
+ if (url === "https://store.example.com/internal")
+ return new Response(null, {
+ status: 302,
+ headers: { location: "http://169.254.169.254/" },
+ });
+ return new Response(null, { status: 404 });
}),
);
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("reads a blocked page through Context and counts its credit", async () => {
const usage = { cloudflareAttempts: 0, contextCredits: 0 };
const result = await readSite("store.example.com", 3, usage);
@@ -145,4 +156,20 @@ describe("readSite on a throttled Shopify store", () => {
]);
expect(usage.contextCredits).toBe(1);
});
+
+ it("does not spend Context credits on a missing page", async () => {
+ const usage = { cloudflareAttempts: 0, contextCredits: 0 };
+
+ await readPages(["https://store.example.com/missing"], 1, usage);
+
+ expect(usage.contextCredits).toBe(0);
+ });
+
+ it("does not send a redirect to a blocked host to Context", async () => {
+ const usage = { cloudflareAttempts: 0, contextCredits: 0 };
+
+ await readPages(["https://store.example.com/internal"], 1, usage);
+
+ expect(usage.contextCredits).toBe(0);
+ });
});
diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.ts
index 972cfffa..0062f90f 100644
--- a/src/server/lib/scrape.ts
+++ b/src/server/lib/scrape.ts
@@ -5,10 +5,7 @@
// gracefully (less text); a Browser Rendering upgrade can slot in behind this
// same interface later.
-import {
- isShopifyHost,
- normalizeAndValidateStartUrl,
-} from "@/server/lib/audit/url-policy";
+import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
import { RenderedPageService } from "@/server/lib/audit/rendered-page";
import { getOptionalEnvValue } from "@/server/lib/runtime-env";
import type { RenderUsage } from "@/shared/audit-rendering";
@@ -58,31 +55,17 @@ export async function readBoundedText(
}
/**
- * Shopify's edge throttles requests from Cloudflare Workers, so a throttled
- * storefront page is read through Context instead. Its credits are added to
- * `usage` for the caller to bill.
+ * With `contextUsage`, a page the plain fetch can't read is read through
+ * Context, because many sites throttle or block Cloudflare Workers (Shopify's
+ * edge, bot protection). Its credits are added to `contextUsage` for the caller
+ * to bill. A missing page (404/410) is not retried there.
*/
-async function readThrottledShopifyPage(
- url: string,
- usage: RenderUsage,
-): Promise<string | null> {
- const contextApiKey = await getOptionalEnvValue("CONTEXT_API_KEY");
- if (!contextApiKey || !(await isShopifyHost(new URL(url).hostname)))
- return null;
- try {
- return (await RenderedPageService.renderPage(url, { contextApiKey, usage }))
- .html;
- } catch {
- return null;
- }
-}
-
async function fetchText(
url: string,
contextUsage?: RenderUsage,
): Promise<string | null> {
+ let current = url;
try {
- let current = url;
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
const response = await fetch(current, {
headers: { "user-agent": USER_AGENT, accept: "text/html,*/*" },
@@ -91,21 +74,37 @@ async function fetchText(
redirect: "manual",
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
- if (response.status === 429 && contextUsage) {
+ if (response.ok) return await readBoundedText(response);
+ const location = response.headers.get("location");
+ if (response.status < 300 || response.status >= 400 || !location) {
await response.body?.cancel();
- return readThrottledShopifyPage(current, contextUsage);
+ if (response.status === 404 || response.status === 410) return null;
+ break;
}
- if (response.status < 300 || response.status >= 400)
- return response.ok ? await readBoundedText(response) : null;
- const location = response.headers.get("location");
- if (!location) return null;
// Re-validates host, blocks private IPs, and does DoH DNS resolution.
- // Throws on a blocked or invalid destination.
- current = await normalizeAndValidateStartUrl(
- new URL(location, current).toString(),
- );
+ // A blocked or invalid destination is not sent to Context either: it
+ // follows redirects itself and would reach the blocked host.
+ try {
+ current = await normalizeAndValidateStartUrl(
+ new URL(location, current).toString(),
+ );
+ } catch {
+ return null;
+ }
}
- return null;
+ } catch {
+ // Timeouts, TLS and network errors fall through to Context.
+ }
+ if (!contextUsage) return null;
+ const contextApiKey = await getOptionalEnvValue("CONTEXT_API_KEY");
+ if (!contextApiKey) return null;
+ try {
+ return (
+ await RenderedPageService.renderPage(current, {
+ contextApiKey,
+ usage: contextUsage,
+ })
+ ).html;
} catch {
return null;
}
@@ -188,7 +187,7 @@ async function scrapePage(
* exact pages (the user's own or a competitor's) rather than asking us to
* discover a site. Each URL is independently run through the SSRF guard, so a
* blocked or unreachable URL is skipped rather than failing the batch.
- * With `contextUsage`, throttled Shopify pages are read through Context.
+ * With `contextUsage`, pages the plain fetch can't read are read through Context.
*/
export async function readPages(
urls: string[],