Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Return 400 when actions are requested but no engines support then (#2704)
Abimael Martellbb258ecb5d8b
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex a802fd8f2..4e2306633 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configurationexport const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);+export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;export const HAS_PROXY = !!config.PROXY_SERVER;diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 31ded4a57..2e851b0d4 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -7,6 +7,7 @@ import {TEST_PRODUCTION,TEST_SELF_HOST,TEST_SUITE_WEBSITE,+ HAS_FIRE_ENGINE,HAS_PLAYWRIGHT,HAS_PROXY,HAS_AI,@@ -232,6 +233,31 @@ describe("Scrape tests", () => {scrapeTimeout,);+ itIf(TEST_SELF_HOST && !HAS_FIRE_ENGINE)(+ "rejects actions when fire-engine is not configured",+ async () => {+ const raw = await scrapeRaw(+ {+ url: "https://example.com",+ timeout: scrapeTimeout,+ actions: [+ {+ type: "wait",+ milliseconds: 1000,+ },+ ],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.code).toBe("SCRAPE_ACTIONS_NOT_SUPPORTED");+ expect(raw.body.error).toContain("Actions are not supported");+ },+ scrapeTimeout,+ );+describe("JSON scrape support", () => {concurrentIf(ALLOW_TEST_SUITE_WEBSITE)("returns parseable JSON",diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 9719c92bf..438f7eb48 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -7,6 +7,7 @@ import {TEST_PRODUCTION,TEST_SELF_HOST,TEST_SUITE_WEBSITE,+ HAS_FIRE_ENGINE,HAS_PLAYWRIGHT,HAS_PROXY,HAS_AI,@@ -278,6 +279,30 @@ describe("Scrape tests", () => {scrapeTimeout,);+ itIf(TEST_SELF_HOST && !HAS_FIRE_ENGINE)(+ "rejects actions when fire-engine is not configured",+ async () => {+ const raw = await scrapeRaw(+ {+ url: "https://example.com",+ actions: [+ {+ type: "wait",+ milliseconds: 1000,+ },+ ],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.code).toBe("SCRAPE_ACTIONS_NOT_SUPPORTED");+ expect(raw.body.error).toContain("Actions are not supported");+ },+ scrapeTimeout,+ );+describeIf(ALLOW_TEST_SUITE_WEBSITE)("JSON scrape support", () => {it.concurrent("returns parseable JSON",diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 20c43a701..656acda0a 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -193,6 +193,14 @@ export async function scrapeController(});}+ if (e.code === "SCRAPE_ACTIONS_NOT_SUPPORTED") {+ return res.status(400).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 033d5a6db..f7856426f 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -296,6 +296,17 @@ export async function scrapeController(});}+ if (e.code === "SCRAPE_ACTIONS_NOT_SUPPORTED") {+ setSpanAttributes(span, {+ "scrape.status_code": 400,+ });+ return res.status(400).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 21936f81c..9dc7ba709 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -1,4 +1,5 @@import {+ ActionsNotSupportedError,CrawlDenialError,ErrorCodes,MapTimeoutError,@@ -44,6 +45,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,SCRAPE_NO_CACHED_DATA: NoCachedDataError,SCRAPE_ACTION_ERROR: ActionError,+ SCRAPE_ACTIONS_NOT_SUPPORTED: ActionsNotSupportedError,SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex fe59dd281..6691096e2 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -18,6 +18,7 @@ export type ErrorCodes =| "SCRAPE_RACED_REDIRECT_ERROR"| "SCRAPE_NO_CACHED_DATA"| "SCRAPE_SITEMAP_ERROR"+ | "SCRAPE_ACTIONS_NOT_SUPPORTED"| "CRAWL_DENIAL"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST";@@ -173,6 +174,25 @@ export class CrawlDenialError extends TransportableError {}}+export class ActionsNotSupportedError extends TransportableError {+ constructor(message: string) {+ super("SCRAPE_ACTIONS_NOT_SUPPORTED", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ActionsNotSupportedError(data.message);+ x.stack = data.stack;+ return x;+ }+}+/*** Error thrown when a job is cancelled (expected flow control, not a real error)* This should not be sent to Sentry as it's expected behavior when a crawl/batch is cancelleddiff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 3d4c9a38f..153ed3633 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -73,7 +73,11 @@ import {AbortManager,AbortManagerThrownError,} from "./lib/abortManager";-import { ScrapeJobTimeoutError, CrawlDenialError } from "../../lib/error";+import {+ ScrapeJobTimeoutError,+ CrawlDenialError,+ ActionsNotSupportedError,+} from "../../lib/error";import { htmlTransform } from "./lib/removeUnwantedElements";import { postprocessors } from "./postprocessors";import { rewriteUrl } from "./lib/rewriteUrl";@@ -504,6 +508,18 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const fallbackList = await buildFallbackList(meta);+ // Check if actions are requested but no engines support them+ if (meta.featureFlags.has("actions")) {+ if (+ fallbackList.length === 0 ||+ fallbackList.every(engine => engine.unsupportedFeatures.has("actions"))+ ) {+ throw new ActionsNotSupportedError(+ "Actions are not supported by any available engines. Actions require Fire Engine (fire-engine) to be enabled.",+ );+ }+ }+setSpanAttributes(span, {"engine.fallback_list": fallbackList.map(f => f.engine).join(","),});