Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Explain Shopify rate limits and read throttled Shopify sites through Context during setup (#954)
Carter Smithf765ced3f0d3
diff --git a/docs/SELF_HOSTING_CLOUDFLARE_OPERATIONS.md b/docs/SELF_HOSTING_CLOUDFLARE_OPERATIONS.mdindex 55182d0c..7fa01819 100644--- a/docs/SELF_HOSTING_CLOUDFLARE_OPERATIONS.md+++ b/docs/SELF_HOSTING_CLOUDFLARE_OPERATIONS.md@@ -36,7 +36,7 @@ Site Audit can load pages in a browser before analyzing them. Turn on **Render JRendering uses [Cloudflare Browser Run](https://developers.cloudflare.com/browser-run/) on your own Cloudflare account. The deploy binds a browser to the audit Worker, so the toggle works with no extra setup. **Rendering needs a paid Workers plan.** The free plan allows one browser action every ten seconds, which is too slow for a crawl. The paid plan includes 10 browser hours a month, then bills $0.09 per browser hour.-Some sites block Cloudflare's browser. To render those pages through [Context.dev](https://www.context.dev) instead, create a Context account, set `CONTEXT_API_KEY` in `.env.selfhost`, and redeploy. OpenSEO calls Context only for pages that Browser Run could not render, and each of those pages uses one Context credit on your Context account.+Some sites block Cloudflare's browser. To render those pages through [Context.dev](https://www.context.dev) instead, create a Context account, set `CONTEXT_API_KEY` in `.env.selfhost`, and redeploy. OpenSEO calls Context only for pages that Browser Run could not render, and for Shopify store pages that Shopify rate limits while project setup or AI Visibility setup reads your site. Each of those pages uses one Context credit on your Context account.Docker deployments have no browser. On Docker, set `CONTEXT_API_KEY` in `.env.local` to render every page through Context. Until the key is set, the **Render JavaScript** toggle stays disabled with a setup note.diff --git a/src/client/features/ai-visibility/AiResearchSetupGate.tsx b/src/client/features/ai-visibility/AiResearchSetupGate.tsxindex 51105611..7d0349cf 100644--- a/src/client/features/ai-visibility/AiResearchSetupGate.tsx+++ b/src/client/features/ai-visibility/AiResearchSetupGate.tsx@@ -4,11 +4,15 @@ import { Sparkles } from "lucide-react";import { Alert, AlertDescription } from "@/client/components/ui/alert";import { Button } from "@/client/components/ui/button";import { Card, CardContent } from "@/client/components/ui/card";+import { Field, FieldLabel } from "@/client/components/ui/field";+import { Textarea } from "@/client/components/ui/textarea";import { projectsQueryOptions } from "@/client/features/projects/projectQueries";import { WebsiteResearchProgress } from "@/client/features/projects/WebsiteResearchProgress";import { WebsiteSetupReview } from "@/client/features/projects/WebsiteSetupReview";import { startAiResearchSetup } from "@/serverFunctions/ai-visibility";+import { updateProjectContext } from "@/serverFunctions/projectContext";import { saveProjectWebsiteSetup } from "@/serverFunctions/projectWebsite";+import { PROSE_MAX_CHARS } from "@/types/schemas/projectContext";import type { SaveProjectWebsiteSetup } from "@/types/schemas/projectWebsite";import { SkeletonPageContent } from "@/client/components/SkeletonPresets";import {@@ -52,6 +56,25 @@ export function AiResearchSetupGate({mutationFn: () => startAiResearchSetup({ data: { projectId } }),onSuccess: (data) => queryClient.setQueryData(setupOptions.queryKey, data),});+ // A site we can't read, usually because of bot protection, leaves the+ // overview to the user. With one saved, setup no longer reads the site.+ const [overview, setOverview] = useState("");+ const saveOverview = useMutation({+ mutationFn: () =>+ updateProjectContext({+ data: {+ projectId,+ updates: [{ section: "business_overview", content: overview.trim() }],+ },+ }),+ onSuccess: () => {+ // Setup can still fail, so the saved overview mustn't wait for it.+ void queryClient.invalidateQueries({+ queryKey: ["projectContext", projectId],+ });+ start.mutate();+ },+ });// Setup writes the tracker, research keywords and a first run. Load what the// AI visibility pages read before setup reads ready, so the page that// started it doesn't flash what it cached before. A failed load drops that@@ -155,13 +178,39 @@ export function AiResearchSetupGate({<Alert variant="destructive"><AlertDescription>{unreadable- ? "We couldn’t read this website. Check the URL in project settings, then try again."+ ? "We couldn’t read this website. Its bot protection may be blocking us. Describe your business below and try again, or check the URL in project settings.": "We couldn’t finish setting up AI visibility. Check your available usage credits, then try again."}</AlertDescription></Alert>)}+ {unreadable && (+ <Field>+ <FieldLabel htmlFor="business-overview">+ What you sell, who buys it, and where+ </FieldLabel>+ <Textarea+ id="business-overview"+ value={overview}+ onChange={(event) => setOverview(event.target.value)}+ maxLength={PROSE_MAX_CHARS}+ rows={4}+ />+ {saveOverview.isError && (+ <p className="text-sm text-destructive">+ We couldn’t save your description. Try again.+ </p>+ )}+ </Field>+ )}<div className="space-y-3">- <Button size="lg" className="w-full" onClick={() => start.mutate()}>+ <Button+ size="lg"+ className="w-full"+ pending={saveOverview.isPending}+ onClick={() =>+ overview.trim() ? saveOverview.mutate() : start.mutate()+ }+ >{failed ? "Try again" : "Start research"}</Button><p className="text-xs leading-relaxed text-muted-foreground">diff --git a/src/client/features/audit/AuditDetail.tsx b/src/client/features/audit/AuditDetail.tsxindex 76d96edc..20c2621b 100644--- a/src/client/features/audit/AuditDetail.tsx+++ b/src/client/features/audit/AuditDetail.tsx@@ -105,15 +105,17 @@ export function AuditDetail({isFailed && resultsQuery.isSuccess && !failedWithResults;const results = resultsQuery.data;// A completed crawl that reached one page or none was blocked by the site,- // unless that page is an app shell, which the results explain instead.- // A failed audit stopped on our side, so it gets the error code instead.+ // unless that page is an app shell or was rate limited, which the results+ // explain instead. A failed audit stopped on our side, so it gets the error+ // code instead.const siteBlocked =isComplete &&status.pagesCrawled <= 1 &&results !== undefined &&!results.issues.some((issue) => issue.issueType === "javascript-rendering-suspected",- );+ ) &&+ !results.pages.some((page) => page.fetchClass === "rate_limited");return (<div className="px-4 py-4 md:px-6 md:py-6 pb-24 md:pb-8 overflow-auto">diff --git a/src/server/billing/researchSpend.ts b/src/server/billing/researchSpend.tsindex 15e6f929..647199b9 100644--- a/src/server/billing/researchSpend.ts+++ b/src/server/billing/researchSpend.ts@@ -7,7 +7,7 @@ import { isHostedServerAuthMode } from "@/server/lib/runtime-env";import type { CreditFeature } from "@/shared/billing-credit-features";export type ResearchSpend = {- provider: "openrouter" | "dataforseo";+ provider: "openrouter" | "dataforseo" | "context";creditFeature: CreditFeature;operation: string;costUsd: number;diff --git a/src/server/features/audit/services/AuditService.crawlerAccess.test.ts b/src/server/features/audit/services/AuditService.crawlerAccess.test.tsindex a2d8787c..ebe960a8 100644--- a/src/server/features/audit/services/AuditService.crawlerAccess.test.ts+++ b/src/server/features/audit/services/AuditService.crawlerAccess.test.ts@@ -128,6 +128,19 @@ describe("startAudit crawler access", () => {expect(params.access).toMatchObject({ host: "store.example.com" });});+ it("records a signed store as Shopify when the probe itself is throttled", async () => {+ // Shopify's edge 429 carries no `powered-by` header.+ vi.mocked(fetch).mockImplementation(() =>+ Promise.resolve(new Response(null, { status: 429 })),+ );++ await AuditService.startAudit(input);++ expect(createAuditMock.mock.calls[0][0].config.sitePlatform).toBe(+ "shopify",+ );+ });+it("does not replay an expired signature", async () => {findCredentialMock.mockResolvedValue([await credentialRow("2020-01-01T00:00:00.000Z"),diff --git a/src/server/features/audit/services/AuditService.ts b/src/server/features/audit/services/AuditService.tsindex 0a7c8382..b9366d72 100644--- a/src/server/features/audit/services/AuditService.ts+++ b/src/server/features/audit/services/AuditService.ts@@ -109,10 +109,12 @@ async function startAudit(input: {lighthouseStrategy,renderJavaScript,// Shopify storefronts answer with `powered-by: Shopify`; knowing this is- // what lets the report explain a throttled crawl instead of shrugging.- sitePlatform: probe.poweredBy?.toLowerCase().includes("shopify")- ? "shopify"- : undefined,+ // what lets the report explain a throttled crawl instead of shrugging. A+ // throttled probe sees no header, but a saved signature is always Shopify's.+ sitePlatform:+ credential || probe.poweredBy?.toLowerCase().includes("shopify")+ ? "shopify"+ : undefined,crawlerCredentialId: credential?.id,};diff --git a/src/server/features/projects/services/websiteResearch.test.ts b/src/server/features/projects/services/websiteResearch.test.tsindex c9f16219..d86cef64 100644--- a/src/server/features/projects/services/websiteResearch.test.ts+++ b/src/server/features/projects/services/websiteResearch.test.ts@@ -44,6 +44,7 @@ vi.mock();import { researchWebsite } from "./websiteResearch";+import { billResearchSpend } from "@/server/billing/researchSpend";import { AppError } from "@/server/lib/errors";import { customer } from "@/server/features/ai-visibility/services/aiVisibilityTestFixtures";@@ -208,4 +209,21 @@ describe("researchWebsite", () => {).rejects.toThrow("Website research was incomplete");expect(search).toHaveBeenCalledOnce();});++ it("bills Context reads of a site that stays unreadable", async () => {+ listSections.mockResolvedValue([]);+ readSite.mockImplementation(+ async (_site, _pages, usage: { contextCredits: number }) => {+ usage.contextCredits += 1;+ return { pages: [] };+ },+ );++ await expect(+ researchWebsite("https://saved.com", project, customer),+ ).rejects.toThrow(AppError);+ expect(billResearchSpend).toHaveBeenCalledWith(customer, [+ expect.objectContaining({ provider: "context", costUsd: 0.0025 }),+ ]);+ });});diff --git a/src/server/features/projects/services/websiteResearch.ts b/src/server/features/projects/services/websiteResearch.tsindex 403ed41c..f8ab7fab 100644--- a/src/server/features/projects/services/websiteResearch.ts+++ b/src/server/features/projects/services/websiteResearch.ts@@ -16,6 +16,10 @@ import {getRequiredEnvValue,} from "@/server/lib/runtime-env";import { readSite } from "@/server/lib/scrape";+import {+ CONTEXT_USD_PER_CREDIT,+ type RenderUsage,+} from "@/shared/audit-rendering";import {MAX_SUGGESTED_KEYWORDS,WEBSITE_UNREADABLE_MESSAGE,@@ -78,12 +82,12 @@ export async function researchWebsite(return websiteResearchSchema.parse(preserved);// Each field is independent: saved context supplies the evidence, and the// model only receives output fields and tools for the missing information.- const site = missingOverview- ? await readSite(project.domain ? `https://${domain}` : website, 3)- : null;- if (site && site.pages.length === 0)- throw new AppError("VALIDATION_ERROR", WEBSITE_UNREADABLE_MESSAGE);const spend: ResearchSpend[] = [];+ // Throttled Shopify pages are read through Context, billed with the research.+ const contextUsage: RenderUsage = {+ cloudflareAttempts: 0,+ contextCredits: 0,+ };const sources = new Set<string>();let searched = false;let searchSucceeded = false;@@ -118,6 +122,15 @@ export async function researchWebsite(: {}),});try {+ const site = missingOverview+ ? await readSite(+ project.domain ? `https://${domain}` : website,+ 3,+ contextUsage,+ )+ : null;+ if (site && site.pages.length === 0)+ throw new AppError("VALIDATION_ERROR", WEBSITE_UNREADABLE_MESSAGE);const result = await generateText({model: buildChatAgentModel(await getRequiredEnvValue("OPENROUTER_API_KEY"),@@ -269,7 +282,8 @@ export async function researchWebsite(scope: "domain",},).apiTarget;- const pages = (await readSite(candidate, 2)).pages;+ const pages = (await readSite(candidate, 2, contextUsage))+ .pages;for (const page of pages) sources.add(page.url);return { domain: competitorDomain, pages };}),@@ -344,6 +358,13 @@ export async function researchWebsite(}return research;} finally {+ if (contextUsage.contextCredits > 0)+ spend.push({+ provider: "context",+ creditFeature: "agent",+ costUsd: contextUsage.contextCredits * CONTEXT_USD_PER_CREDIT,+ operation: "project_website_research",+ });await billResearchSpend(customer, spend);}}diff --git a/src/server/lib/audit/rendered-page.ts b/src/server/lib/audit/rendered-page.tsindex 9a026834..a948cf07 100644--- a/src/server/lib/audit/rendered-page.ts+++ b/src/server/lib/audit/rendered-page.ts@@ -17,7 +17,8 @@ type RenderOptions = {// Context key is optional on self-hosted Cloudflare deployments.browser?: Pick<BrowserRun, "quickAction">;contextApiKey?: string;- auditId: string;+ /** Logged with each attempt; reads outside an audit have none. */+ auditId?: string;// Counts every provider attempt, including rejected ones: both providers// bill a request whether or not the HTML turned out to be usable.usage: RenderUsage;diff --git a/src/server/lib/audit/url-policy.ts b/src/server/lib/audit/url-policy.tsindex cc806970..0b86e5b8 100644--- a/src/server/lib/audit/url-policy.ts+++ b/src/server/lib/audit/url-policy.ts@@ -187,6 +187,20 @@ async function hostnameResolvesToBlockedAddress(}}+/**+ * Shopify storefronts, custom domains included, resolve into Shopify's+ * 23.227.38.0/24. This identifies a store even when its edge answers with a+ * bare 429 that carries no `powered-by` header.+ */+export async function isShopifyHost(hostname: string): Promise<boolean> {+ try {+ const addresses = await resolveAddressRecords(normalizeHost(hostname), "A");+ return addresses.some((address) => address.startsWith("23.227.38."));+ } catch {+ return false;+ }+}+/*** Synchronous SSRF check for URLs discovered mid-crawl (links, redirect* targets, sitemap entries). Blocks non-http(s) schemes, private/loopback IPdiff --git a/src/server/lib/scrape.test.ts b/src/server/lib/scrape.test.tsindex 6cfea683..1c33cfba 100644--- a/src/server/lib/scrape.test.ts+++ b/src/server/lib/scrape.test.ts@@ -1,6 +1,10 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { readPages, readSite } from "@/server/lib/scrape";+vi.mock("@/server/lib/runtime-env", () => ({+ getOptionalEnvValue: async () => "context-test-key",+}));+describe("readSite SSRF guard", () => {beforeEach(() => {vi.stubGlobal("fetch", vi.fn());@@ -99,3 +103,46 @@ describe("readSite", () => {expect(result.blocked).toBe(true);});});++describe("readSite on a throttled Shopify store", () => {+ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("reads the page through Context and counts its credit", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn(async (input: string | URL) => {+ const url = String(input);+ if (url.includes("dns-query"))+ // In Shopify's range, so the 429 below is Shopify's edge.+ return Response.json({+ Status: 0,+ Answer: [{ type: 1, data: "23.227.38.65" }],+ });+ if (url.startsWith("https://api.context.dev/"))+ return Response.json({+ success: true,+ type: "html",+ html: "<title>Store</title><p>Linen sheets</p>",+ metadata: { finalUrl: "https://store.example.com/" },+ finalDOMState: "loaded",+ key_metadata: { credits_consumed: 1 },+ });+ return new Response(null, { status: 429 });+ }),+ );+ const usage = { cloudflareAttempts: 0, contextCredits: 0 };++ const result = await readSite("store.example.com", 3, usage);++ expect(result.pages).toEqual([+ {+ url: "https://store.example.com/",+ title: "Store",+ text: "Store Linen sheets",+ },+ ]);+ expect(usage.contextCredits).toBe(1);+ });+});diff --git a/src/server/lib/scrape.ts b/src/server/lib/scrape.tsindex deff560c..66824ff1 100644--- a/src/server/lib/scrape.ts+++ b/src/server/lib/scrape.ts@@ -5,7 +5,13 @@// gracefully (less text); a Browser Rendering upgrade can slot in behind this// same interface later.-import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";+import {+ isShopifyHost,+ normalizeAndValidateStartUrl,+} from "@/server/lib/audit/url-policy";+import { RenderedPageService } from "@/server/lib/audit/rendered-page";+import { getOptionalEnvValue } from "@/server/lib/runtime-env";+import type { RenderUsage } from "@/shared/audit-rendering";const MAX_PAGES = 5;const PER_PAGE_CHAR_LIMIT = 4000;@@ -49,7 +55,30 @@ async function readBoundedText(response: Response): Promise<string | null> {return result;}-async function fetchText(url: string): Promise<string | null> {+/**+ * Shopify's edge throttles requests from Cloudflare Workers, so a throttled+ * storefront page is read through Context instead. Its credits are added to+ * `usage` for the caller to bill.+ */+async function readThrottledShopifyPage(+ url: string,+ usage: RenderUsage,+): Promise<string | null> {+ const contextApiKey = await getOptionalEnvValue("CONTEXT_API_KEY");+ if (!contextApiKey || !(await isShopifyHost(new URL(url).hostname)))+ return null;+ try {+ return (await RenderedPageService.renderPage(url, { contextApiKey, usage }))+ .html;+ } catch {+ return null;+ }+}++async function fetchText(+ url: string,+ contextUsage?: RenderUsage,+): Promise<string | null> {try {let current = url;for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {@@ -60,6 +89,10 @@ async function fetchText(url: string): Promise<string | null> {redirect: "manual",signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),});+ if (response.status === 429 && contextUsage) {+ await response.body?.cancel();+ return readThrottledShopifyPage(current, contextUsage);+ }if (response.status < 300 || response.status >= 400)return response.ok ? await readBoundedText(response) : null;const location = response.headers.get("location");@@ -131,8 +164,11 @@ function decodeEntities(value: string): string {/** Fetches one (already-validated) URL and shapes it as a page, or null if it* couldn't be read or yielded no text. */-async function scrapePage(url: string): Promise<ScrapedPage | null> {- const html = await fetchText(url);+async function scrapePage(+ url: string,+ contextUsage?: RenderUsage,+): Promise<ScrapedPage | null> {+ const html = await fetchText(url, contextUsage);if (!html) {return null;}@@ -150,10 +186,12 @@ async function scrapePage(url: string): Promise<ScrapedPage | null> {* exact pages (the user's own or a competitor's) rather than asking us to* discover a site. Each URL is independently run through the SSRF guard, so a* blocked or unreachable URL is skipped rather than failing the batch.+ * With `contextUsage`, throttled Shopify pages are read through Context.*/export async function readPages(urls: string[],maxPages: number = MAX_PAGES,+ contextUsage?: RenderUsage,): Promise<SiteReadResult> {const pages: ScrapedPage[] = [];for (const rawUrl of urls.slice(0, maxPages)) {@@ -164,7 +202,7 @@ export async function readPages(} catch {continue; // blocked or unparseable URL}- const page = await scrapePage(url);+ const page = await scrapePage(url, contextUsage);if (page) {pages.push(page);}@@ -204,12 +242,13 @@ export async function discoverSiteUrls(export async function readSite(domain: string,maxPages: number = MAX_PAGES,+ contextUsage?: RenderUsage,): Promise<SiteReadResult> {const discovered = await discoverSiteUrls(domain, maxPages);if (discovered.blocked) {return { pages: [], blocked: true };}- const site = await readPages(discovered.urls, maxPages);+ const site = await readPages(discovered.urls, maxPages, contextUsage);const root = new URL(discovered.urls[0]);if (!site.blocked || root.hostname.startsWith("www.")) return site;// Some sites only serve their www host (e.g. the bare domain's TLS@@ -217,5 +256,5 @@ export async function readSite(root.hostname = `www.${root.hostname}`;// The setter ignores invalid hosts, so IP literals keep their original host.if (!root.hostname.startsWith("www.")) return site;- return readSite(root.toString(), maxPages);+ return readSite(root.toString(), maxPages, contextUsage);}diff --git a/src/shared/audit-rendering.ts b/src/shared/audit-rendering.tsindex 29ec3b47..b47eb478 100644--- a/src/shared/audit-rendering.ts+++ b/src/shared/audit-rendering.ts@@ -11,7 +11,7 @@ const CLOUDFLARE_USD_PER_BROWSER_HOUR = 0.09;const CLOUDFLARE_USD_PER_ATTEMPT =(CLOUDFLARE_RENDER_TIMEOUT_MS / 3_600_000) * CLOUDFLARE_USD_PER_BROWSER_HOUR;// Context.dev list price on the Developer plan. One scrape is one credit.-const CONTEXT_USD_PER_CREDIT = 0.0025;+export const CONTEXT_USD_PER_CREDIT = 0.0025;/** Provider units a rendered audit used, counted per crawl chunk. */export type RenderUsage = {diff --git a/src/shared/crawler-access.test.ts b/src/shared/crawler-access.test.tsindex 848eab25..a5e7f332 100644--- a/src/shared/crawler-access.test.ts+++ b/src/shared/crawler-access.test.ts@@ -1,6 +1,7 @@import { describe, expect, it } from "vitest";import {crawlerHeadersFor,+ normalizeCrawlerHost,shopifyCrawlerHeaders,} from "@/shared/crawler-access";@@ -32,3 +33,10 @@ describe("crawlerHeadersFor", () => {);});});++describe("normalizeCrawlerHost", () => {+ it("rejects a scheme whose host https requests would read differently", () => {+ // Under https, "5.6" is the IPv4 address 5.0.0.6.+ expect(normalizeCrawlerHost("h://5.6")).toBeNull();+ });+});diff --git a/src/shared/crawler-access.ts b/src/shared/crawler-access.tsindex d3c8e1c0..7b9f1cd4 100644--- a/src/shared/crawler-access.ts+++ b/src/shared/crawler-access.ts@@ -49,14 +49,17 @@ export function normalizeCrawlerHost(input: string): string | null {const raw = input.trim().toLowerCase();if (!raw) return null;- let hostname: string;+ let url: URL;try {- hostname = new URL(raw.includes("://") ? raw : `https://${raw}`).hostname;+ url = new URL(raw.includes("://") ? raw : `https://${raw}`);} catch {return null;}+ // Other schemes parse hosts differently ("h://5.6" keeps "5.6", which https+ // reads as 5.0.0.6), so the saved host would never match a request.+ if (url.protocol !== "https:" && url.protocol !== "http:") return null;- const host = hostname.replace(/\.$/, "");+ const host = url.hostname.replace(/\.$/, "");if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(host)) return null;return host;}