Skip to content

Open source alternative to Semrush and Ahrefs

Render throttled Shopify pages and ask for Render JavaScript when signatures don't help (#957)

Carter Smithdc7eb0372a91
diff --git a/docs/maintainers/SITE_AUDIT_RENDERING.md b/docs/maintainers/SITE_AUDIT_RENDERING.md
index 5793d290..28e08b83 100644
--- a/docs/maintainers/SITE_AUDIT_RENDERING.md
+++ b/docs/maintainers/SITE_AUDIT_RENDERING.md
@@ -56,6 +56,23 @@ rendered document over 1 MiB keeps its first 1 MiB, the same bound the direct
crawl reads. The audit never audits the unrendered shell in its place, and paid
calls are never retried automatically.
+## Throttled Shopify storefronts
+
+Shopify's edge rate limits requests from Cloudflare Workers with a 429, often
+even when they carry a crawler signature, and backing off rarely helps. When the
+audited site is a Shopify store (its `powered-by` header, a saved crawler
+signature, or a host in Shopify's address range) and the audit renders
+JavaScript, a page whose direct fetch returns 429 goes straight to the
+renderers instead of the shared backoff. Browser Run is usually throttled too;
+Context fetches from outside Cloudflare and reads the page. A page neither
+renderer reads stays `rate_limited` and enters the shared backoff, so a store
+that stays unreadable pauses and then stops the crawl as before.
+
+An unrendered audit of a Shopify store backs off as before. Its report asks for
+a crawler signature first, since Shopify sometimes lets a signed crawl through.
+Once a signed, unrendered crawl is still rate limited, the report asks for a
+new audit with Render JavaScript turned on instead.
+
## Billing
Rates live in `src/shared/audit-rendering.ts`, shared by the launch form, the
diff --git a/src/client/features/audit/results/ResultsView.tsx b/src/client/features/audit/results/ResultsView.tsx
index c1a13af5..234b8fd1 100644
--- a/src/client/features/audit/results/ResultsView.tsx
+++ b/src/client/features/audit/results/ResultsView.tsx
@@ -64,8 +64,7 @@ export function ResultsView({
[pages],
);
- // Shopify's own crawler-access signature is a real fix for a throttled or
- // refused crawl, so it replaces the generic advice for those stores.
+ // Shopify stores get their own notice instead of the generic advice.
const shopifyLimited =
audit.config.sitePlatform === "shopify" &&
(blockedCount > 0 || rateLimitedCount > 0 || crawlStopped);
@@ -95,7 +94,11 @@ export function ResultsView({
return (
<>
{shopifyLimited && (
- <ShopifyCrawlWarning projectId={projectId} audit={audit} />
+ <ShopifyCrawlWarning
+ projectId={projectId}
+ audit={audit}
+ throttled={rateLimitedCount > 0 || crawlStopped}
+ />
)}
{!shopifyLimited && !siteBlocked && blockedCount > 0 && (
diff --git a/src/client/features/audit/results/ShopifyCrawlWarning.tsx b/src/client/features/audit/results/ShopifyCrawlWarning.tsx
index 6a386dff..04fff7f1 100644
--- a/src/client/features/audit/results/ShopifyCrawlWarning.tsx
+++ b/src/client/features/audit/results/ShopifyCrawlWarning.tsx
@@ -12,6 +12,7 @@ import {
AlertTitle,
} from "@/client/components/ui/alert";
import { Button } from "@/client/components/ui/button";
+import { ShopifyRateLimitAlert } from "@/client/features/audit/results/ShopifyRateLimitAlert";
import type { AuditResultsData } from "@/client/features/audit/results/types";
import { extractHostname } from "@/client/features/audit/shared";
import { startAudit } from "@/serverFunctions/audit";
@@ -24,17 +25,22 @@ import {
/**
* Shown instead of the generic crawl warnings when a Shopify storefront
* throttled or refused the crawl. Shopify's own fix is a crawler-access
- * signature the merchant creates in admin, so the report asks for it here.
+ * signature the merchant creates in admin, so the report asks for it first.
*
* The setup steps only show while the domain has no usable signature. Once
- * one is saved they collapse to a single line with the re-run button.
+ * one is saved they collapse to a single line with the re-run button. An
+ * unrendered crawl Shopify still rate limited with the signature asks for
+ * rendering instead.
*/
export function ShopifyCrawlWarning({
projectId,
audit,
+ throttled,
}: {
projectId: string;
audit: AuditResultsData["audit"];
+ /** Shopify answered with 429s, not only refusals. */
+ throttled: boolean;
}) {
const navigate = useNavigate();
const host = extractHostname(audit.startUrl);
@@ -106,6 +112,14 @@ export function ShopifyCrawlWarning({
if (liveCredential) {
const signedAndStillLimited = liveCredential.id === usedCredentialId;
+ if (signedAndStillLimited && throttled && !audit.config.renderJavaScript) {
+ return (
+ <ShopifyRateLimitAlert
+ projectId={projectId}
+ startUrl={audit.startUrl}
+ />
+ );
+ }
return (
<Alert variant={signedAndStillLimited ? "warning" : "success"}>
{signedAndStillLimited ? <ShieldAlert /> : <ShieldCheck />}
diff --git a/src/client/features/audit/results/ShopifyRateLimitAlert.tsx b/src/client/features/audit/results/ShopifyRateLimitAlert.tsx
new file mode 100644
index 00000000..b014ac4d
--- /dev/null
+++ b/src/client/features/audit/results/ShopifyRateLimitAlert.tsx
@@ -0,0 +1,48 @@
+import { Link } from "@tanstack/react-router";
+import { ShieldAlert } from "lucide-react";
+import {
+ Alert,
+ AlertDescription,
+ AlertTitle,
+} from "@/client/components/ui/alert";
+import { Button } from "@/client/components/ui/button";
+
+/**
+ * Shown when Shopify rate limited an unrendered crawl that carried a crawler
+ * signature. Shopify's edge can still throttle Cloudflare Workers then, and a
+ * rendered audit reads throttled pages from outside Cloudflare.
+ */
+export function ShopifyRateLimitAlert({
+ projectId,
+ startUrl,
+}: {
+ projectId: string;
+ startUrl: string;
+}) {
+ return (
+ <Alert variant="warning">
+ <ShieldAlert />
+ <AlertTitle>We ran into an error with your Shopify domain.</AlertTitle>
+ <AlertDescription>
+ Shopify is globally rate limiting, so this report is incomplete. To
+ complete the audit, start a new one with Render JavaScript turned on.
+ </AlertDescription>
+ <div className="col-start-2 mt-2">
+ <Button
+ variant="outline"
+ size="sm"
+ nativeButton={false}
+ render={
+ <Link
+ to="/p/$projectId/audit"
+ params={{ projectId }}
+ search={{ url: startUrl }}
+ />
+ }
+ >
+ Start a new audit
+ </Button>
+ </div>
+ </Alert>
+ );
+}
diff --git a/src/server/features/audit/services/AuditService.rendering.test.ts b/src/server/features/audit/services/AuditService.rendering.test.ts
index 2788017c..57892eed 100644
--- a/src/server/features/audit/services/AuditService.rendering.test.ts
+++ b/src/server/features/audit/services/AuditService.rendering.test.ts
@@ -27,6 +27,7 @@ vi.mock("@/server/lib/audit/rendering-billing", () => ({
vi.mock("@/server/lib/audit/url-policy", () => ({
normalizeAndValidateStartUrl: async (url: string) => url,
resolveStartUrlRedirects: async (url: string) => ({ url }),
+ isShopifyHost: async () => false,
}));
vi.mock("@/server/features/audit/services/CrawlerCredentialService", () => ({
CrawlerCredentialService: {
diff --git a/src/server/features/audit/services/AuditService.ts b/src/server/features/audit/services/AuditService.ts
index b9366d72..05238edd 100644
--- a/src/server/features/audit/services/AuditService.ts
+++ b/src/server/features/audit/services/AuditService.ts
@@ -28,6 +28,7 @@ import {
type LighthouseStrategy,
} from "@/server/lib/audit/types";
import {
+ isShopifyHost,
normalizeAndValidateStartUrl,
resolveStartUrlRedirects,
} from "@/server/lib/audit/url-policy";
@@ -109,10 +110,13 @@ async function startAudit(input: {
lighthouseStrategy,
renderJavaScript,
// Shopify storefronts answer with `powered-by: Shopify`; knowing this is
- // what lets the report explain a throttled crawl instead of shrugging. A
- // throttled probe sees no header, but a saved signature is always Shopify's.
+ // what lets a rendered crawl read throttled pages and the report explain
+ // an unrendered one. A throttled probe sees no header, but a saved
+ // signature is always Shopify's, and so is a host in Shopify's range.
sitePlatform:
- credential || probe.poweredBy?.toLowerCase().includes("shopify")
+ credential ||
+ probe.poweredBy?.toLowerCase().includes("shopify") ||
+ (await isShopifyHost(startHost))
? "shopify"
: undefined,
crawlerCredentialId: credential?.id,
diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts
index a09e49ba..35af939a 100644
--- a/src/server/workflows/site-audit-workflow-helpers.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.ts
@@ -35,6 +35,8 @@ async function fetchPage(
url: string,
throttle: CrawlThrottle,
access: CrawlerAccess | null | undefined,
+ /** Hand a 429 straight back instead of backing off. */
+ skipBackoff: boolean,
) {
for (let attempt = 1; ; attempt++) {
if (!(await throttle.ready())) return null;
@@ -65,6 +67,7 @@ async function fetchPage(
await throttle.recovered();
return result;
}
+ if (skipBackoff) return result;
const retry = await throttle.backoff(
attempt,
@@ -86,13 +89,20 @@ export async function crawlPage(
/** Crawler-access headers for the audited host, when the org has one. */
access?: CrawlerAccess | null;
render?: (url: string) => Promise<RenderedPage>;
+ /**
+ * Shopify's edge throttles Cloudflare Workers before the store sees the
+ * request, and waiting rarely helps. A rendered audit of a Shopify store
+ * reads a throttled page through the renderer instead of backing off.
+ */
+ renderThrottled?: boolean;
} = {},
): Promise<CrawledPageResult | null> {
const { access, render } = options;
+ const renderThrottled = Boolean(render && options.renderThrottled);
const startTime = Date.now();
try {
- const fetched = await fetchPage(url, throttle, access);
+ const fetched = await fetchPage(url, throttle, access, renderThrottled);
if (!fetched) return null;
const { response, responseTimeMs, rateLimited } = fetched;
let statusCode = response.status;
@@ -120,7 +130,7 @@ export async function crawlPage(
}
const contentType = response.headers.get("content-type") ?? "";
- const isHtml = contentType.includes("text/html");
+ let isHtml = contentType.includes("text/html");
// Cap what we read: the first 1 MiB still contains the SEO metadata and
// navigation needed by the audit in normal documents.
let body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
@@ -130,27 +140,34 @@ export async function crawlPage(
body.slice(0, 4_000),
);
- // Rendering replaces the body of a readable HTML page, or of a bot
- // challenge, with what the browser loaded. Redirects, non-HTML files,
- // origin errors, login walls and rate limits keep their existing paths.
- // The direct response's timing and headers stay authoritative.
+ // Rendering replaces the body of a readable HTML page, a bot challenge, or
+ // a throttled Shopify page with what the browser loaded. Redirects,
+ // non-HTML files, origin errors, login walls and other rate limits keep
+ // their existing paths. The direct response's timing and headers stay
+ // authoritative.
const challenged = fetchClass === "blocked" && statusCode !== 401;
+ const throttled = renderThrottled && statusCode === 429;
if (
render &&
- isHtml &&
- (challenged || (fetchClass === "ok" && statusCode < 400))
+ (throttled ||
+ (isHtml && (challenged || (fetchClass === "ok" && statusCode < 400))))
) {
try {
const page = await render(url);
body = page.html;
- // A challenge's status was never the page's. Browser Run reports the
- // status it loaded; Context returns only pages that loaded.
- if (challenged) statusCode = page.status ?? 200;
+ isHtml = true;
+ // A challenge's or a 429's status was never the page's. Browser Run
+ // reports the status it loaded; Context returns only pages that loaded.
+ if (challenged || throttled) statusCode = page.status ?? 200;
// The renderer can be challenged where the direct fetch was not.
fetchClass = classifyFetch(statusCode, false, body.slice(0, 4_000));
} catch (error) {
// A challenge that neither renderer passed stays blocked.
- if (!challenged) throw error;
+ if (!challenged && !throttled) throw error;
+ // A throttled page neither renderer read stays rate limited, and
+ // pauses or stops the crawl like any other 429.
+ if (throttled)
+ await throttle.backoff(1, response.headers.get("retry-after"));
}
}
diff --git a/src/server/workflows/siteAuditWorkflowCrawl.test.ts b/src/server/workflows/siteAuditWorkflowCrawl.test.ts
index 47f1db49..6b38110b 100644
--- a/src/server/workflows/siteAuditWorkflowCrawl.test.ts
+++ b/src/server/workflows/siteAuditWorkflowCrawl.test.ts
@@ -104,7 +104,11 @@ afterEach(() => {
vi.useRealTimers();
});
-function crawl(maxPages = 100, renderJavaScript = false) {
+function crawl(
+ maxPages = 100,
+ renderJavaScript = false,
+ sitePlatform?: "shopify",
+) {
urls = Array.from({ length: maxPages }, (_, i) => `${ORIGIN}/${i}`);
return runCrawlPhase(
{
@@ -120,6 +124,7 @@ function crawl(maxPages = 100, renderJavaScript = false) {
maxPages,
seededCount: maxPages,
renderJavaScript,
+ sitePlatform,
robots: parseRobotsTxt(ORIGIN, ""),
renderUsage,
},
@@ -317,3 +322,24 @@ describe("rendering usage", () => {
});
});
});
+
+describe("throttled Shopify pages", () => {
+ it("are rendered instead of waiting when the audit renders JavaScript", async () => {
+ serve(() => 429);
+ mocks.renderPage.mockResolvedValue({ html: HTML, status: null });
+ const run = crawl(10, true, "shopify");
+ await vi.runAllTimersAsync();
+ expect(await run).toEqual({ pagesCrawled: 10, completed: true });
+ expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);
+ // The first backoff would have waited 30 seconds.
+ expect(Date.now()).toBeLessThan(30_000);
+ });
+
+ it("stop the crawl like any 429 when no renderer reads them", async () => {
+ serve(() => 429);
+ mocks.renderPage.mockRejectedValue(new Error("Unable to render this page"));
+ const run = crawl(100, true, "shopify");
+ await vi.runAllTimersAsync();
+ expect(await run).toMatchObject({ completed: false, rateLimited: true });
+ });
+});
diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.ts
index 75dd09a8..2eff9e77 100644
--- a/src/server/workflows/siteAuditWorkflowCrawl.ts
+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts
@@ -1,7 +1,7 @@
import { env, type WorkflowStep } from "cloudflare:workers";
import { NonRetryableError } from "cloudflare:workflows";
import type { RobotsResult } from "@/server/lib/audit/discovery";
-import type { CrawledPageResult } from "@/server/lib/audit/types";
+import type { AuditConfig, CrawledPageResult } from "@/server/lib/audit/types";
import { isSameOrigin } from "@/server/lib/audit/url-utils";
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
@@ -94,6 +94,7 @@ type CrawlPhaseParams = {
/** Frontier size after discovery seeding (from the discover-urls step). */
seededCount: number;
renderJavaScript?: boolean;
+ sitePlatform?: AuditConfig["sitePlatform"];
/**
* Rendering totals across chunks. Mutated as each chunk step returns, so a
* failed audit can still settle what its completed chunks rendered.
@@ -317,7 +318,11 @@ async function runCrawlChunk(
entry.depth,
entry.inSitemap,
throttle,
- { access: input.access, render },
+ {
+ access: input.access,
+ render,
+ renderThrottled: input.sitePlatform === "shopify",
+ },
)
.then((page) => {
if (!page) {
diff --git a/src/server/workflows/siteAuditWorkflowPhases.ts b/src/server/workflows/siteAuditWorkflowPhases.ts
index a677a196..c2adc788 100644
--- a/src/server/workflows/siteAuditWorkflowPhases.ts
+++ b/src/server/workflows/siteAuditWorkflowPhases.ts
@@ -67,7 +67,6 @@ export async function runAuditPhases(
startUrl,
config,
access,
- renderUsage,
} = params;
const origin = getOrigin(startUrl);
const maxPages = config.maxPages;
@@ -85,15 +84,13 @@ export async function runAuditPhases(
// desync the frontier from already-persisted crawl batches).
const robots = parseRobotsTxt(origin, discovery.robotsText);
const crawl = await runCrawlPhase(step, {
- auditId,
- workflowInstanceId,
+ ...params,
origin,
maxPages,
robots,
seededCount: discovery.seededCount,
renderJavaScript: config.renderJavaScript,
- renderUsage,
- access,
+ sitePlatform: config.sitePlatform,
});
await runLighthousePhase(step, {
auditId,