Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(scrapeURL): remove the stealth-skip and 429 waterfall experiments (#5565)

rowinsg4591e46a8e09
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index a76c17016..80cbb929a 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -79,7 +79,6 @@ export class NoEnginesLeftError extends TransportableError {
export class AddFeatureError extends Error {
public featureFlags: FeatureFlag[];
- public statusCode?: number;
public pdfPrefetch: Meta["pdfPrefetch"];
public documentPrefetch: Meta["documentPrefetch"];
public imagePrefetch: Meta["imagePrefetch"];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 1595b32dc..9a60d3525 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -800,9 +800,7 @@ async function scrapeURLLoopIter(
length: engineResult.html?.trim().length ?? 0,
},
);
- throw Object.assign(new AddFeatureError(["stealthProxy"]), {
- statusCode: engineResult.statusCode,
- });
+ throw new AddFeatureError(["stealthProxy"]);
}
// NOTE: TODO: what to do when status code is bad is tough...
@@ -1009,11 +1007,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error: error.error,
});
if (testArm) {
- if (
- experiment === "skip-enhanced-retry-after-site-protection"
- ) {
- remainingEngines.shift();
- }
enginePromises = enginePromises.filter(
x => x.engine !== error.engine,
);
diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts
index 8693904ec..657939f34 100644
--- a/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts
+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.test.ts
@@ -1,4 +1,4 @@
-import { AddFeatureError, EngineError, SiteError } from "./error";
+import { SiteError } from "./error";
import {
inWaterfallExperimentArm,
waterfallExperimentFor,
@@ -32,81 +32,18 @@ describe("waterfallExperimentFor", () => {
),
).toBeUndefined();
});
-
- it("takes a site_protection failure on the first enhanced attempt", () => {
- const blocked = new EngineError("Scrape job failed", {
- cause: { status: { failureReason: "site_protection" } },
- });
- expect(
- waterfallExperimentFor(
- "fire-engine;chrome-cdp;stealth",
- blocked,
- "fire-engine(retry);chrome-cdp;stealth",
- ),
- ).toBe("skip-enhanced-retry-after-site-protection");
- expect(
- waterfallExperimentFor(
- "fire-engine;chrome-cdp;stealth",
- new EngineError("Scrape job failed", { cause: { status: {} } }),
- "fire-engine(retry);chrome-cdp;stealth",
- ),
- ).toBeUndefined();
- });
-
- it("takes a 429 escalation on the first basic attempt only", () => {
- const escalation = (statusCode: number) =>
- Object.assign(new AddFeatureError(["stealthProxy"]), { statusCode });
- expect(
- waterfallExperimentFor(
- "fire-engine;chrome-cdp",
- escalation(429),
- "fire-engine(retry);chrome-cdp",
- ),
- ).toBe("basic-retry-on-429");
- expect(
- waterfallExperimentFor(
- "fire-engine;chrome-cdp",
- escalation(403),
- "fire-engine(retry);chrome-cdp",
- ),
- ).toBeUndefined();
- expect(
- waterfallExperimentFor(
- "fire-engine(retry);chrome-cdp",
- escalation(429),
- "fire-engine;chrome-cdp;stealth",
- ),
- ).toBeUndefined();
- });
});
describe("inWaterfallExperimentArm", () => {
it("is stable per scrape and close to the rollout", () => {
const ids = Array.from({ length: 4000 }, (_, i) => `scrape-${i}`);
- const share = (
- experiment: Parameters<typeof inWaterfallExperimentArm>[1],
- ) =>
- ids.filter(id => inWaterfallExperimentArm(id, experiment)).length /
- ids.length;
+ const share =
+ ids.filter(id => inWaterfallExperimentArm(id, "tunnel-error-retry"))
+ .length / ids.length;
expect(inWaterfallExperimentArm("scrape-1", "tunnel-error-retry")).toBe(
inWaterfallExperimentArm("scrape-1", "tunnel-error-retry"),
);
- expect(share("tunnel-error-retry")).toBeGreaterThan(0.45);
- expect(share("tunnel-error-retry")).toBeLessThan(0.55);
- expect(share("skip-enhanced-retry-after-site-protection")).toBeGreaterThan(
- 0.16,
- );
- expect(share("skip-enhanced-retry-after-site-protection")).toBeLessThan(
- 0.24,
- );
- expect(share("basic-retry-on-429")).toBeGreaterThan(0.45);
- expect(share("basic-retry-on-429")).toBeLessThan(0.55);
- const both = ids.filter(
- id =>
- inWaterfallExperimentArm(id, "tunnel-error-retry") &&
- inWaterfallExperimentArm(id, "basic-retry-on-429"),
- ).length;
- expect(both / ids.length).toBeGreaterThan(0.2);
- expect(both / ids.length).toBeLessThan(0.3);
+ expect(share).toBeGreaterThan(0.45);
+ expect(share).toBeLessThan(0.55);
});
});
diff --git a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts
index b546fa288..0e4a9081c 100644
--- a/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts
+++ b/apps/api/src/scraper/scrapeURL/waterfallExperiments.ts
@@ -1,11 +1,9 @@
import { createHash } from "node:crypto";
import type { Engine } from "./engines";
-import { AddFeatureError, EngineError, SiteError } from "./error";
+import { SiteError } from "./error";
const ROLLOUTS = {
"tunnel-error-retry": 0.5,
- "skip-enhanced-retry-after-site-protection": 0.2,
- "basic-retry-on-429": 0.5,
} as const;
export type WaterfallExperiment = keyof typeof ROLLOUTS;
@@ -25,23 +23,6 @@ export function waterfallExperimentFor(
) {
return "tunnel-error-retry";
}
- if (
- engine === "fire-engine;chrome-cdp;stealth" &&
- nextEngine === "fire-engine(retry);chrome-cdp;stealth" &&
- error instanceof EngineError &&
- (error.cause as { status?: { failureReason?: string } } | undefined)?.status
- ?.failureReason === "site_protection"
- ) {
- return "skip-enhanced-retry-after-site-protection";
- }
- if (
- engine === "fire-engine;chrome-cdp" &&
- nextEngine === "fire-engine(retry);chrome-cdp" &&
- error instanceof AddFeatureError &&
- error.statusCode === 429
- ) {
- return "basic-retry-on-429";
- }
return undefined;
}