Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): return js returns from f-e (FIR-1535) (#1385)
Gergő Móricz46048bc94d43
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 9d9109fbc..459e5e560 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -541,6 +541,10 @@ export type Document = {actions?: {screenshots?: string[];scrapes?: ScrapeActionContent[];+ javascriptReturns?: {+ type: string,+ value: unknown+ }[];};metadata: {title?: string;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex ac6fabfd6..47322ef04 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -42,6 +42,36 @@ const successSchema = z.object({}).array().optional(),+ actionResults: z.union([+ z.object({+ idx: z.number(),+ type: z.literal("screenshot"),+ result: z.object({+ path: z.string(),+ }),+ }),+ z.object({+ idx: z.number(),+ type: z.literal("scrape"),+ result: z.union([+ z.object({+ url: z.string(),+ html: z.string(),+ }),+ z.object({+ url: z.string(),+ accessibility: z.string(),+ }),+ ]),+ }),+ z.object({+ idx: z.number(),+ type: z.literal("executeJavascript"),+ result: z.object({+ return: z.string(),+ }),+ }),+ ]).array().optional(),// chrome-cdp only -- file download handlerfile: z@@ -138,7 +168,7 @@ export async function fireEngineCheckStatus(} else if (typeof status.error === "string" &&// TODO: improve this later- status.error.includes("Element")+ (status.error.includes("Element") || status.error.includes("Javascript execution failed"))) {throw new ActionError(status.error.split("Error: ")[1]);} else {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex c21d9f90e..93596fcee 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -274,6 +274,7 @@ export async function scrapeURLWithFireEngineChromeCDP(actions: {screenshots: response.screenshots ?? [],scrapes: response.actionContent ?? [],+ javascriptReturns: (response.actionResults ?? []).filter(x => x.type === "executeJavascript").map(x => JSON.parse((x.result as any as { return: string }).return)),},}: {}),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 0a688fc6a..ab2fe79ba 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -103,6 +103,10 @@ export type EngineScrapeResult = {actions?: {screenshots: string[];scrapes: ScrapeActionContent[];+ javascriptReturns: {+ type: string;+ value: unknown+ }[];};};diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 5cc9d1195..11fb8d747 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -141,6 +141,14 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchemexport interface ActionsResult {screenshots: string[];+ scrapes: ({+ url: string;+ html: string;+ })[];+ javascriptReturns: {+ type: string;+ value: unknown+ }[];}/**