Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): add waterfall retry experiments (#5454)
rowinsg304881f46e20
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex b0703656b..76baf6acf 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -63,6 +63,7 @@ export class NoEnginesLeftError extends TransportableError {export class AddFeatureError extends Error {public featureFlags: FeatureFlag[];+ public statusCode?: number;public pdfPrefetch: Meta["pdfPrefetch"];public documentPrefetch: Meta["documentPrefetch"];public imagePrefetch: Meta["imagePrefetch"];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0d220cd78..699f4d3c0 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -63,6 +63,10 @@ import {ExchangeRefusedError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";+import {+ inWaterfallExperimentArm,+ waterfallExperimentFor,+} from "./waterfallExperiments";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";import { shouldCheckRobots } from "./shouldCheckRobots";@@ -794,7 +798,9 @@ async function scrapeURLLoopIter(length: engineResult.html?.trim().length ?? 0,},);- throw new AddFeatureError(["stealthProxy"]);+ throw Object.assign(new AddFeatureError(["stealthProxy"]), {+ statusCode: engineResult.statusCode,+ });}// NOTE: TODO: what to do when status code is bad is tough...@@ -985,6 +991,36 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {break;} catch (error) {if (error instanceof WrappedEngineError) {+ const nextEngine = remainingEngines[0]?.engine;+ const experiment = waterfallExperimentFor(+ error.engine,+ error.error,+ nextEngine,+ );+ if (experiment) {+ const testArm = inWaterfallExperimentArm(meta.id, experiment);+ meta.logger.info("Waterfall experiment decision", {+ experiment,+ arm: testArm ? "test" : "control",+ engine: error.engine,+ nextEngine,+ error: error.error,+ });+ if (testArm) {+ if (+ experiment === "skip-enhanced-retry-after-site-protection"+ ) {+ remainingEngines.shift();+ }+ enginePromises = enginePromises.filter(+ x => x.engine !== error.engine,+ );+ if (enginePromises.length === 0) {+ break;+ }+ continue;+ }+ }if (error.engine === "x-twitter") {meta.logger.warn("X/Twitter scrape failed fatally.", {error: error.error,diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.tsnew file mode 100644index 000000000..8693904ec--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts@@ -0,0 +1,112 @@+import { AddFeatureError, EngineError, SiteError } from "./error";+import {+ inWaterfallExperimentArm,+ waterfallExperimentFor,+} from "./waterfallExperiments";++describe("waterfallExperimentFor", () => {+ it("takes a tunnel error on a first attempt that has its retry engine next", () => {+ const error = new SiteError("net::ERR_TUNNEL_CONNECTION_FAILED");+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp",+ error,+ "fire-engine(retry);chrome-cdp",+ ),+ ).toBe("tunnel-error-retry");+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp;stealth",+ error,+ "fire-engine(retry);chrome-cdp;stealth",+ ),+ ).toBe("tunnel-error-retry");+ expect(+ waterfallExperimentFor("fire-engine(retry);chrome-cdp", error, undefined),+ ).toBeUndefined();+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp",+ new SiteError("ERR_CONNECTION_REFUSED"),+ "fire-engine(retry);chrome-cdp",+ ),+ ).toBeUndefined();+ });++ it("takes a site_protection failure on the first enhanced attempt", () => {+ const blocked = new EngineError("Scrape job failed", {+ cause: { status: { failureReason: "site_protection" } },+ });+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp;stealth",+ blocked,+ "fire-engine(retry);chrome-cdp;stealth",+ ),+ ).toBe("skip-enhanced-retry-after-site-protection");+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp;stealth",+ new EngineError("Scrape job failed", { cause: { status: {} } }),+ "fire-engine(retry);chrome-cdp;stealth",+ ),+ ).toBeUndefined();+ });++ it("takes a 429 escalation on the first basic attempt only", () => {+ const escalation = (statusCode: number) =>+ Object.assign(new AddFeatureError(["stealthProxy"]), { statusCode });+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp",+ escalation(429),+ "fire-engine(retry);chrome-cdp",+ ),+ ).toBe("basic-retry-on-429");+ expect(+ waterfallExperimentFor(+ "fire-engine;chrome-cdp",+ escalation(403),+ "fire-engine(retry);chrome-cdp",+ ),+ ).toBeUndefined();+ expect(+ waterfallExperimentFor(+ "fire-engine(retry);chrome-cdp",+ escalation(429),+ "fire-engine;chrome-cdp;stealth",+ ),+ ).toBeUndefined();+ });+});++describe("inWaterfallExperimentArm", () => {+ it("is stable per scrape and close to the rollout", () => {+ const ids = Array.from({ length: 4000 }, (_, i) => `scrape-${i}`);+ const share = (+ experiment: Parameters<typeof inWaterfallExperimentArm>[1],+ ) =>+ ids.filter(id => inWaterfallExperimentArm(id, experiment)).length /+ ids.length;+ expect(inWaterfallExperimentArm("scrape-1", "tunnel-error-retry")).toBe(+ inWaterfallExperimentArm("scrape-1", "tunnel-error-retry"),+ );+ expect(share("tunnel-error-retry")).toBeGreaterThan(0.45);+ expect(share("tunnel-error-retry")).toBeLessThan(0.55);+ expect(share("skip-enhanced-retry-after-site-protection")).toBeGreaterThan(+ 0.16,+ );+ expect(share("skip-enhanced-retry-after-site-protection")).toBeLessThan(+ 0.24,+ );+ expect(share("basic-retry-on-429")).toBeGreaterThan(0.45);+ expect(share("basic-retry-on-429")).toBeLessThan(0.55);+ const both = ids.filter(+ id =>+ inWaterfallExperimentArm(id, "tunnel-error-retry") &&+ inWaterfallExperimentArm(id, "basic-retry-on-429"),+ ).length;+ expect(both / ids.length).toBeGreaterThan(0.2);+ expect(both / ids.length).toBeLessThan(0.3);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.tsnew file mode 100644index 000000000..b546fa288--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts@@ -0,0 +1,59 @@+import { createHash } from "node:crypto";+import type { Engine } from "./engines";+import { AddFeatureError, EngineError, SiteError } from "./error";++const ROLLOUTS = {+ "tunnel-error-retry": 0.5,+ "skip-enhanced-retry-after-site-protection": 0.2,+ "basic-retry-on-429": 0.5,+} as const;++export type WaterfallExperiment = keyof typeof ROLLOUTS;++export function waterfallExperimentFor(+ engine: Engine,+ error: unknown,+ nextEngine: Engine | undefined,+): WaterfallExperiment | undefined {+ if (+ error instanceof SiteError &&+ error.errorCode.includes("ERR_TUNNEL_CONNECTION_FAILED") &&+ ((engine === "fire-engine;chrome-cdp" &&+ nextEngine === "fire-engine(retry);chrome-cdp") ||+ (engine === "fire-engine;chrome-cdp;stealth" &&+ nextEngine === "fire-engine(retry);chrome-cdp;stealth"))+ ) {+ return "tunnel-error-retry";+ }+ if (+ engine === "fire-engine;chrome-cdp;stealth" &&+ nextEngine === "fire-engine(retry);chrome-cdp;stealth" &&+ error instanceof EngineError &&+ (error.cause as { status?: { failureReason?: string } } | undefined)?.status+ ?.failureReason === "site_protection"+ ) {+ return "skip-enhanced-retry-after-site-protection";+ }+ if (+ engine === "fire-engine;chrome-cdp" &&+ nextEngine === "fire-engine(retry);chrome-cdp" &&+ error instanceof AddFeatureError &&+ error.statusCode === 429+ ) {+ return "basic-retry-on-429";+ }+ return undefined;+}++export function inWaterfallExperimentArm(+ scrapeId: string,+ experiment: WaterfallExperiment,+): boolean {+ const bucket =+ createHash("sha256")+ .update(`${scrapeId}:${experiment}`)+ .digest()+ .readUInt32BE(0) /+ 2 ** 32;+ return bucket < ROLLOUTS[experiment];+}