Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/fire-engine/chrome-cdp): handle file downloads
Gergő Móriczf15ef0e7582e
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 6f65db98e..e02e9dbb7 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node";import { z } from "zod";import { robustFetch } from "../../lib/fetch";-import { ActionError, EngineError, SiteError } from "../../error";+import { ActionError, EngineError, SiteError, UnsupportedFileError } from "../../error";const successSchema = z.object({jobId: z.string(),@@ -35,6 +35,12 @@ const successSchema = z.object({}).array().optional(),++ // chrome-cdp only -- file download handler+ file: z.object({+ name: z.string(),+ content: z.string(),+ }).optional().or(z.null()),});export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;@@ -111,6 +117,11 @@ export async function fireEngineCheckStatus(status.error.includes("Chrome error: ")) {throw new SiteError(status.error.split("Chrome error: ")[1]);+ } else if (+ typeof status.error === "string" &&+ status.error.includes("File size exceeds")+ ) {+ throw new UnsupportedFileError("File size exceeds " + status.error.split("File size exceeds ")[1]);} else if (typeof status.error === "string" &&// TODO: improve this laterdiff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex d753465dc..aa869836e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -13,7 +13,7 @@ import {FireEngineCheckStatusSuccess,StillProcessingError,} from "./checkStatus";-import { ActionError, EngineError, SiteError, TimeoutError } from "../../error";+import { ActionError, EngineError, SiteError, TimeoutError, UnsupportedFileError } from "../../error";import * as Sentry from "@sentry/node";import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";@@ -71,7 +71,8 @@ async function performFireEngineScrape<} else if (error instanceof EngineError ||error instanceof SiteError ||- error instanceof ActionError+ error instanceof ActionError ||+ error instanceof UnsupportedFileError) {logger.debug("Fire-engine scrape job failed.", {error,@@ -91,6 +92,19 @@ async function performFireEngineScrape<await new Promise((resolve) => setTimeout(resolve, 250));}+ specialtyScrapeCheck(+ logger.child({+ method: "performFireEngineScrape/specialtyScrapeCheck",+ }),+ status.responseHeaders,+ );++ if (status.file) {+ const content = status.file.content;+ delete status.file;+ status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag+ }+return status;}@@ -160,13 +174,6 @@ export async function scrapeURLWithFireEngineChromeCDP(timeout,);- specialtyScrapeCheck(- meta.logger.child({- method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck",- }),- response.responseHeaders,- );-if (meta.options.formats.includes("screenshot") ||meta.options.formats.includes("screenshot@fullPage")@@ -241,13 +248,6 @@ export async function scrapeURLWithFireEnginePlaywright(timeout,);- specialtyScrapeCheck(- meta.logger.child({- method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck",- }),- response.responseHeaders,- );-if (!response.url) {meta.logger.warn("Fire-engine did not return the response's URL", {response,@@ -301,13 +301,6 @@ export async function scrapeURLWithFireEngineTLSClient(timeout,);- specialtyScrapeCheck(- meta.logger.child({- method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck",- }),- response.responseHeaders,- );-if (!response.url) {meta.logger.warn("Fire-engine did not return the response's URL", {response,diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 689f90c80..bff3a492e 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -64,3 +64,11 @@ export class ActionError extends Error {this.code = code;}}++export class UnsupportedFileError extends Error {+ public reason: string;+ constructor(reason: string) {+ super("Scrape resulted in unsupported file: " + reason);+ this.reason = reason;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 1df812bdb..130ef9ee1 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -19,6 +19,7 @@ import {RemoveFeatureError,SiteError,TimeoutError,+ UnsupportedFileError,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -292,6 +293,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw error;} else if (error instanceof ActionError) {throw error;+ } else if (error instanceof UnsupportedFileError) {+ throw error;} else {Sentry.captureException(error);meta.logger.info(@@ -414,6 +417,8 @@ export async function scrapeURL(meta.logger.warn("scrapeURL: Site failed to load in browser", { error });} else if (error instanceof ActionError) {meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });+ } else if (error instanceof UnsupportedFileError) {+ meta.logger.warn("scrapeURL: Tried to scrape unsupported file", { error });} else {Sentry.captureException(error);meta.logger.error("scrapeURL: Unexpected error happened", { error });