Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Render throttled Shopify pages and ask for Render JavaScript when signatures don't help (#957)
Carter Smithdc7eb0372a91
diff --git a/docs/maintainers/SITE_AUDIT_RENDERING.md b/docs/maintainers/SITE_AUDIT_RENDERING.mdindex 5793d290..28e08b83 100644--- a/docs/maintainers/SITE_AUDIT_RENDERING.md+++ b/docs/maintainers/SITE_AUDIT_RENDERING.md@@ -56,6 +56,23 @@ rendered document over 1 MiB keeps its first 1 MiB, the same bound the directcrawl reads. The audit never audits the unrendered shell in its place, and paidcalls are never retried automatically.+## Throttled Shopify storefronts++Shopify's edge rate limits requests from Cloudflare Workers with a 429, often+even when they carry a crawler signature, and backing off rarely helps. When the+audited site is a Shopify store (its `powered-by` header, a saved crawler+signature, or a host in Shopify's address range) and the audit renders+JavaScript, a page whose direct fetch returns 429 goes straight to the+renderers instead of the shared backoff. Browser Run is usually throttled too;+Context fetches from outside Cloudflare and reads the page. A page neither+renderer reads stays `rate_limited` and enters the shared backoff, so a store+that stays unreadable pauses and then stops the crawl as before.++An unrendered audit of a Shopify store backs off as before. Its report asks for+a crawler signature first, since Shopify sometimes lets a signed crawl through.+Once a signed, unrendered crawl is still rate limited, the report asks for a+new audit with Render JavaScript turned on instead.+## BillingRates live in `src/shared/audit-rendering.ts`, shared by the launch form, thediff --git a/src/client/features/audit/results/ResultsView.tsx b/src/client/features/audit/results/ResultsView.tsxindex c1a13af5..234b8fd1 100644--- a/src/client/features/audit/results/ResultsView.tsx+++ b/src/client/features/audit/results/ResultsView.tsx@@ -64,8 +64,7 @@ export function ResultsView({[pages],);- // Shopify's own crawler-access signature is a real fix for a throttled or- // refused crawl, so it replaces the generic advice for those stores.+ // Shopify stores get their own notice instead of the generic advice.const shopifyLimited =audit.config.sitePlatform === "shopify" &&(blockedCount > 0 || rateLimitedCount > 0 || crawlStopped);@@ -95,7 +94,11 @@ export function ResultsView({return (<>{shopifyLimited && (- <ShopifyCrawlWarning projectId={projectId} audit={audit} />+ <ShopifyCrawlWarning+ projectId={projectId}+ audit={audit}+ throttled={rateLimitedCount > 0 || crawlStopped}+ />)}{!shopifyLimited && !siteBlocked && blockedCount > 0 && (diff --git a/src/client/features/audit/results/ShopifyCrawlWarning.tsx b/src/client/features/audit/results/ShopifyCrawlWarning.tsxindex 6a386dff..04fff7f1 100644--- a/src/client/features/audit/results/ShopifyCrawlWarning.tsx+++ b/src/client/features/audit/results/ShopifyCrawlWarning.tsx@@ -12,6 +12,7 @@ import {AlertTitle,} from "@/client/components/ui/alert";import { Button } from "@/client/components/ui/button";+import { ShopifyRateLimitAlert } from "@/client/features/audit/results/ShopifyRateLimitAlert";import type { AuditResultsData } from "@/client/features/audit/results/types";import { extractHostname } from "@/client/features/audit/shared";import { startAudit } from "@/serverFunctions/audit";@@ -24,17 +25,22 @@ import {/*** Shown instead of the generic crawl warnings when a Shopify storefront* throttled or refused the crawl. Shopify's own fix is a crawler-access- * signature the merchant creates in admin, so the report asks for it here.+ * signature the merchant creates in admin, so the report asks for it first.** The setup steps only show while the domain has no usable signature. Once- * one is saved they collapse to a single line with the re-run button.+ * one is saved they collapse to a single line with the re-run button. An+ * unrendered crawl Shopify still rate limited with the signature asks for+ * rendering instead.*/export function ShopifyCrawlWarning({projectId,audit,+ throttled,}: {projectId: string;audit: AuditResultsData["audit"];+ /** Shopify answered with 429s, not only refusals. */+ throttled: boolean;}) {const navigate = useNavigate();const host = extractHostname(audit.startUrl);@@ -106,6 +112,14 @@ export function ShopifyCrawlWarning({if (liveCredential) {const signedAndStillLimited = liveCredential.id === usedCredentialId;+ if (signedAndStillLimited && throttled && !audit.config.renderJavaScript) {+ return (+ <ShopifyRateLimitAlert+ projectId={projectId}+ startUrl={audit.startUrl}+ />+ );+ }return (<Alert variant={signedAndStillLimited ? "warning" : "success"}>{signedAndStillLimited ? <ShieldAlert /> : <ShieldCheck />}diff --git a/src/client/features/audit/results/ShopifyRateLimitAlert.tsx b/src/client/features/audit/results/ShopifyRateLimitAlert.tsxnew file mode 100644index 00000000..b014ac4d--- /dev/null+++ b/src/client/features/audit/results/ShopifyRateLimitAlert.tsx@@ -0,0 +1,48 @@+import { Link } from "@tanstack/react-router";+import { ShieldAlert } from "lucide-react";+import {+ Alert,+ AlertDescription,+ AlertTitle,+} from "@/client/components/ui/alert";+import { Button } from "@/client/components/ui/button";++/**+ * Shown when Shopify rate limited an unrendered crawl that carried a crawler+ * signature. Shopify's edge can still throttle Cloudflare Workers then, and a+ * rendered audit reads throttled pages from outside Cloudflare.+ */+export function ShopifyRateLimitAlert({+ projectId,+ startUrl,+}: {+ projectId: string;+ startUrl: string;+}) {+ return (+ <Alert variant="warning">+ <ShieldAlert />+ <AlertTitle>We ran into an error with your Shopify domain.</AlertTitle>+ <AlertDescription>+ Shopify is globally rate limiting, so this report is incomplete. To+ complete the audit, start a new one with Render JavaScript turned on.+ </AlertDescription>+ <div className="col-start-2 mt-2">+ <Button+ variant="outline"+ size="sm"+ nativeButton={false}+ render={+ <Link+ to="/p/$projectId/audit"+ params={{ projectId }}+ search={{ url: startUrl }}+ />+ }+ >+ Start a new audit+ </Button>+ </div>+ </Alert>+ );+}diff --git a/src/server/features/audit/services/AuditService.rendering.test.ts b/src/server/features/audit/services/AuditService.rendering.test.tsindex 2788017c..57892eed 100644--- a/src/server/features/audit/services/AuditService.rendering.test.ts+++ b/src/server/features/audit/services/AuditService.rendering.test.ts@@ -27,6 +27,7 @@ vi.mock("@/server/lib/audit/rendering-billing", () => ({vi.mock("@/server/lib/audit/url-policy", () => ({normalizeAndValidateStartUrl: async (url: string) => url,resolveStartUrlRedirects: async (url: string) => ({ url }),+ isShopifyHost: async () => false,}));vi.mock("@/server/features/audit/services/CrawlerCredentialService", () => ({CrawlerCredentialService: {diff --git a/src/server/features/audit/services/AuditService.ts b/src/server/features/audit/services/AuditService.tsindex b9366d72..05238edd 100644--- a/src/server/features/audit/services/AuditService.ts+++ b/src/server/features/audit/services/AuditService.ts@@ -28,6 +28,7 @@ import {type LighthouseStrategy,} from "@/server/lib/audit/types";import {+ isShopifyHost,normalizeAndValidateStartUrl,resolveStartUrlRedirects,} from "@/server/lib/audit/url-policy";@@ -109,10 +110,13 @@ async function startAudit(input: {lighthouseStrategy,renderJavaScript,// Shopify storefronts answer with `powered-by: Shopify`; knowing this is- // what lets the report explain a throttled crawl instead of shrugging. A- // throttled probe sees no header, but a saved signature is always Shopify's.+ // what lets a rendered crawl read throttled pages and the report explain+ // an unrendered one. A throttled probe sees no header, but a saved+ // signature is always Shopify's, and so is a host in Shopify's range.sitePlatform:- credential || probe.poweredBy?.toLowerCase().includes("shopify")+ credential ||+ probe.poweredBy?.toLowerCase().includes("shopify") ||+ (await isShopifyHost(startHost))? "shopify": undefined,crawlerCredentialId: credential?.id,diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.tsindex a09e49ba..35af939a 100644--- a/src/server/workflows/site-audit-workflow-helpers.ts+++ b/src/server/workflows/site-audit-workflow-helpers.ts@@ -35,6 +35,8 @@ async function fetchPage(url: string,throttle: CrawlThrottle,access: CrawlerAccess | null | undefined,+ /** Hand a 429 straight back instead of backing off. */+ skipBackoff: boolean,) {for (let attempt = 1; ; attempt++) {if (!(await throttle.ready())) return null;@@ -65,6 +67,7 @@ async function fetchPage(await throttle.recovered();return result;}+ if (skipBackoff) return result;const retry = await throttle.backoff(attempt,@@ -86,13 +89,20 @@ export async function crawlPage(/** Crawler-access headers for the audited host, when the org has one. */access?: CrawlerAccess | null;render?: (url: string) => Promise<RenderedPage>;+ /**+ * Shopify's edge throttles Cloudflare Workers before the store sees the+ * request, and waiting rarely helps. A rendered audit of a Shopify store+ * reads a throttled page through the renderer instead of backing off.+ */+ renderThrottled?: boolean;} = {},): Promise<CrawledPageResult | null> {const { access, render } = options;+ const renderThrottled = Boolean(render && options.renderThrottled);const startTime = Date.now();try {- const fetched = await fetchPage(url, throttle, access);+ const fetched = await fetchPage(url, throttle, access, renderThrottled);if (!fetched) return null;const { response, responseTimeMs, rateLimited } = fetched;let statusCode = response.status;@@ -120,7 +130,7 @@ export async function crawlPage(}const contentType = response.headers.get("content-type") ?? "";- const isHtml = contentType.includes("text/html");+ let isHtml = contentType.includes("text/html");// Cap what we read: the first 1 MiB still contains the SEO metadata and// navigation needed by the audit in normal documents.let body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";@@ -130,27 +140,34 @@ export async function crawlPage(body.slice(0, 4_000),);- // Rendering replaces the body of a readable HTML page, or of a bot- // challenge, with what the browser loaded. Redirects, non-HTML files,- // origin errors, login walls and rate limits keep their existing paths.- // The direct response's timing and headers stay authoritative.+ // Rendering replaces the body of a readable HTML page, a bot challenge, or+ // a throttled Shopify page with what the browser loaded. Redirects,+ // non-HTML files, origin errors, login walls and other rate limits keep+ // their existing paths. The direct response's timing and headers stay+ // authoritative.const challenged = fetchClass === "blocked" && statusCode !== 401;+ const throttled = renderThrottled && statusCode === 429;if (render &&- isHtml &&- (challenged || (fetchClass === "ok" && statusCode < 400))+ (throttled ||+ (isHtml && (challenged || (fetchClass === "ok" && statusCode < 400))))) {try {const page = await render(url);body = page.html;- // A challenge's status was never the page's. Browser Run reports the- // status it loaded; Context returns only pages that loaded.- if (challenged) statusCode = page.status ?? 200;+ isHtml = true;+ // A challenge's or a 429's status was never the page's. Browser Run+ // reports the status it loaded; Context returns only pages that loaded.+ if (challenged || throttled) statusCode = page.status ?? 200;// The renderer can be challenged where the direct fetch was not.fetchClass = classifyFetch(statusCode, false, body.slice(0, 4_000));} catch (error) {// A challenge that neither renderer passed stays blocked.- if (!challenged) throw error;+ if (!challenged && !throttled) throw error;+ // A throttled page neither renderer read stays rate limited, and+ // pauses or stops the crawl like any other 429.+ if (throttled)+ await throttle.backoff(1, response.headers.get("retry-after"));}}diff --git a/src/server/workflows/siteAuditWorkflowCrawl.test.ts b/src/server/workflows/siteAuditWorkflowCrawl.test.tsindex 47f1db49..6b38110b 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.test.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.test.ts@@ -104,7 +104,11 @@ afterEach(() => {vi.useRealTimers();});-function crawl(maxPages = 100, renderJavaScript = false) {+function crawl(+ maxPages = 100,+ renderJavaScript = false,+ sitePlatform?: "shopify",+) {urls = Array.from({ length: maxPages }, (_, i) => `${ORIGIN}/${i}`);return runCrawlPhase({@@ -120,6 +124,7 @@ function crawl(maxPages = 100, renderJavaScript = false) {maxPages,seededCount: maxPages,renderJavaScript,+ sitePlatform,robots: parseRobotsTxt(ORIGIN, ""),renderUsage,},@@ -317,3 +322,24 @@ describe("rendering usage", () => {});});});++describe("throttled Shopify pages", () => {+ it("are rendered instead of waiting when the audit renders JavaScript", async () => {+ serve(() => 429);+ mocks.renderPage.mockResolvedValue({ html: HTML, status: null });+ const run = crawl(10, true, "shopify");+ await vi.runAllTimersAsync();+ expect(await run).toEqual({ pagesCrawled: 10, completed: true });+ expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);+ // The first backoff would have waited 30 seconds.+ expect(Date.now()).toBeLessThan(30_000);+ });++ it("stop the crawl like any 429 when no renderer reads them", async () => {+ serve(() => 429);+ mocks.renderPage.mockRejectedValue(new Error("Unable to render this page"));+ const run = crawl(100, true, "shopify");+ await vi.runAllTimersAsync();+ expect(await run).toMatchObject({ completed: false, rateLimited: true });+ });+});diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.tsindex 75dd09a8..2eff9e77 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts@@ -1,7 +1,7 @@import { env, type WorkflowStep } from "cloudflare:workers";import { NonRetryableError } from "cloudflare:workflows";import type { RobotsResult } from "@/server/lib/audit/discovery";-import type { CrawledPageResult } from "@/server/lib/audit/types";+import type { AuditConfig, CrawledPageResult } from "@/server/lib/audit/types";import { isSameOrigin } from "@/server/lib/audit/url-utils";import { isCrawlableUrl } from "@/server/lib/audit/url-policy";import { deterministicAuditRowId } from "@/server/lib/audit/ids";@@ -94,6 +94,7 @@ type CrawlPhaseParams = {/** Frontier size after discovery seeding (from the discover-urls step). */seededCount: number;renderJavaScript?: boolean;+ sitePlatform?: AuditConfig["sitePlatform"];/*** Rendering totals across chunks. Mutated as each chunk step returns, so a* failed audit can still settle what its completed chunks rendered.@@ -317,7 +318,11 @@ async function runCrawlChunk(entry.depth,entry.inSitemap,throttle,- { access: input.access, render },+ {+ access: input.access,+ render,+ renderThrottled: input.sitePlatform === "shopify",+ },).then((page) => {if (!page) {diff --git a/src/server/workflows/siteAuditWorkflowPhases.ts b/src/server/workflows/siteAuditWorkflowPhases.tsindex a677a196..c2adc788 100644--- a/src/server/workflows/siteAuditWorkflowPhases.ts+++ b/src/server/workflows/siteAuditWorkflowPhases.ts@@ -67,7 +67,6 @@ export async function runAuditPhases(startUrl,config,access,- renderUsage,} = params;const origin = getOrigin(startUrl);const maxPages = config.maxPages;@@ -85,15 +84,13 @@ export async function runAuditPhases(// desync the frontier from already-persisted crawl batches).const robots = parseRobotsTxt(origin, discovery.robotsText);const crawl = await runCrawlPhase(step, {- auditId,- workflowInstanceId,+ ...params,origin,maxPages,robots,seededCount: discovery.seededCount,renderJavaScript: config.renderJavaScript,- renderUsage,- access,+ sitePlatform: config.sitePlatform,});await runLighthousePhase(step, {auditId,