Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): add waterfall retry experiments (#5454)

rowinsg304881f46e20
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index b0703656b..76baf6acf 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -63,6 +63,7 @@ export class NoEnginesLeftError extends TransportableError {
export class AddFeatureError extends Error {
public featureFlags: FeatureFlag[];
+ public statusCode?: number;
public pdfPrefetch: Meta["pdfPrefetch"];
public documentPrefetch: Meta["documentPrefetch"];
public imagePrefetch: Meta["imagePrefetch"];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0d220cd78..699f4d3c0 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -63,6 +63,10 @@ import {
ExchangeRefusedError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
+import {
+ inWaterfallExperimentArm,
+ waterfallExperimentFor,
+} from "./waterfallExperiments";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
import { shouldCheckRobots } from "./shouldCheckRobots";
@@ -794,7 +798,9 @@ async function scrapeURLLoopIter(
length: engineResult.html?.trim().length ?? 0,
},
);
- throw new AddFeatureError(["stealthProxy"]);
+ throw Object.assign(new AddFeatureError(["stealthProxy"]), {
+ statusCode: engineResult.statusCode,
+ });
}
// NOTE: TODO: what to do when status code is bad is tough...
@@ -985,6 +991,36 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
break;
} catch (error) {
if (error instanceof WrappedEngineError) {
+ const nextEngine = remainingEngines[0]?.engine;
+ const experiment = waterfallExperimentFor(
+ error.engine,
+ error.error,
+ nextEngine,
+ );
+ if (experiment) {
+ const testArm = inWaterfallExperimentArm(meta.id, experiment);
+ meta.logger.info("Waterfall experiment decision", {
+ experiment,
+ arm: testArm ? "test" : "control",
+ engine: error.engine,
+ nextEngine,
+ error: error.error,
+ });
+ if (testArm) {
+ if (
+ experiment === "skip-enhanced-retry-after-site-protection"
+ ) {
+ remainingEngines.shift();
+ }
+ enginePromises = enginePromises.filter(
+ x => x.engine !== error.engine,
+ );
+ if (enginePromises.length === 0) {
+ break;
+ }
+ continue;
+ }
+ }
if (error.engine === "x-twitter") {
meta.logger.warn("X/Twitter scrape failed fatally.", {
error: error.error,
diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts
new file mode 100644
index 000000000..8693904ec
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts
@@ -0,0 +1,112 @@
+import { AddFeatureError, EngineError, SiteError } from "./error";
+import {
+ inWaterfallExperimentArm,
+ waterfallExperimentFor,
+} from "./waterfallExperiments";
+
+describe("waterfallExperimentFor", () => {
+ it("takes a tunnel error on a first attempt that has its retry engine next", () => {
+ const error = new SiteError("net::ERR_TUNNEL_CONNECTION_FAILED");
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp",
+ error,
+ "fire-engine(retry);chrome-cdp",
+ ),
+ ).toBe("tunnel-error-retry");
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp;stealth",
+ error,
+ "fire-engine(retry);chrome-cdp;stealth",
+ ),
+ ).toBe("tunnel-error-retry");
+ expect(
+ waterfallExperimentFor("fire-engine(retry);chrome-cdp", error, undefined),
+ ).toBeUndefined();
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp",
+ new SiteError("ERR_CONNECTION_REFUSED"),
+ "fire-engine(retry);chrome-cdp",
+ ),
+ ).toBeUndefined();
+ });
+
+ it("takes a site_protection failure on the first enhanced attempt", () => {
+ const blocked = new EngineError("Scrape job failed", {
+ cause: { status: { failureReason: "site_protection" } },
+ });
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp;stealth",
+ blocked,
+ "fire-engine(retry);chrome-cdp;stealth",
+ ),
+ ).toBe("skip-enhanced-retry-after-site-protection");
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp;stealth",
+ new EngineError("Scrape job failed", { cause: { status: {} } }),
+ "fire-engine(retry);chrome-cdp;stealth",
+ ),
+ ).toBeUndefined();
+ });
+
+ it("takes a 429 escalation on the first basic attempt only", () => {
+ const escalation = (statusCode: number) =>
+ Object.assign(new AddFeatureError(["stealthProxy"]), { statusCode });
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp",
+ escalation(429),
+ "fire-engine(retry);chrome-cdp",
+ ),
+ ).toBe("basic-retry-on-429");
+ expect(
+ waterfallExperimentFor(
+ "fire-engine;chrome-cdp",
+ escalation(403),
+ "fire-engine(retry);chrome-cdp",
+ ),
+ ).toBeUndefined();
+ expect(
+ waterfallExperimentFor(
+ "fire-engine(retry);chrome-cdp",
+ escalation(429),
+ "fire-engine;chrome-cdp;stealth",
+ ),
+ ).toBeUndefined();
+ });
+});
+
+describe("inWaterfallExperimentArm", () => {
+ it("is stable per scrape and close to the rollout", () => {
+ const ids = Array.from({ length: 4000 }, (_, i) => `scrape-${i}`);
+ const share = (
+ experiment: Parameters<typeof inWaterfallExperimentArm>[1],
+ ) =>
+ ids.filter(id => inWaterfallExperimentArm(id, experiment)).length /
+ ids.length;
+ expect(inWaterfallExperimentArm("scrape-1", "tunnel-error-retry")).toBe(
+ inWaterfallExperimentArm("scrape-1", "tunnel-error-retry"),
+ );
+ expect(share("tunnel-error-retry")).toBeGreaterThan(0.45);
+ expect(share("tunnel-error-retry")).toBeLessThan(0.55);
+ expect(share("skip-enhanced-retry-after-site-protection")).toBeGreaterThan(
+ 0.16,
+ );
+ expect(share("skip-enhanced-retry-after-site-protection")).toBeLessThan(
+ 0.24,
+ );
+ expect(share("basic-retry-on-429")).toBeGreaterThan(0.45);
+ expect(share("basic-retry-on-429")).toBeLessThan(0.55);
+ const both = ids.filter(
+ id =>
+ inWaterfallExperimentArm(id, "tunnel-error-retry") &&
+ inWaterfallExperimentArm(id, "basic-retry-on-429"),
+ ).length;
+ expect(both / ids.length).toBeGreaterThan(0.2);
+ expect(both / ids.length).toBeLessThan(0.3);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts
new file mode 100644
index 000000000..b546fa288
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts
@@ -0,0 +1,59 @@
+import { createHash } from "node:crypto";
+import type { Engine } from "./engines";
+import { AddFeatureError, EngineError, SiteError } from "./error";
+
+const ROLLOUTS = {
+ "tunnel-error-retry": 0.5,
+ "skip-enhanced-retry-after-site-protection": 0.2,
+ "basic-retry-on-429": 0.5,
+} as const;
+
+export type WaterfallExperiment = keyof typeof ROLLOUTS;
+
+export function waterfallExperimentFor(
+ engine: Engine,
+ error: unknown,
+ nextEngine: Engine | undefined,
+): WaterfallExperiment | undefined {
+ if (
+ error instanceof SiteError &&
+ error.errorCode.includes("ERR_TUNNEL_CONNECTION_FAILED") &&
+ ((engine === "fire-engine;chrome-cdp" &&
+ nextEngine === "fire-engine(retry);chrome-cdp") ||
+ (engine === "fire-engine;chrome-cdp;stealth" &&
+ nextEngine === "fire-engine(retry);chrome-cdp;stealth"))
+ ) {
+ return "tunnel-error-retry";
+ }
+ if (
+ engine === "fire-engine;chrome-cdp;stealth" &&
+ nextEngine === "fire-engine(retry);chrome-cdp;stealth" &&
+ error instanceof EngineError &&
+ (error.cause as { status?: { failureReason?: string } } | undefined)?.status
+ ?.failureReason === "site_protection"
+ ) {
+ return "skip-enhanced-retry-after-site-protection";
+ }
+ if (
+ engine === "fire-engine;chrome-cdp" &&
+ nextEngine === "fire-engine(retry);chrome-cdp" &&
+ error instanceof AddFeatureError &&
+ error.statusCode === 429
+ ) {
+ return "basic-retry-on-429";
+ }
+ return undefined;
+}
+
+export function inWaterfallExperimentArm(
+ scrapeId: string,
+ experiment: WaterfallExperiment,
+): boolean {
+ const bucket =
+ createHash("sha256")
+ .update(`${scrapeId}:${experiment}`)
+ .digest()
+ .readUInt32BE(0) /
+ 2 ** 32;
+ return bucket < ROLLOUTS[experiment];
+}