Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): return js returns from f-e (FIR-1535) (#1385)

Gergő Móricz46048bc94d43
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 9d9109fbc..459e5e560 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -541,6 +541,10 @@ export type Document = {
actions?: {
screenshots?: string[];
scrapes?: ScrapeActionContent[];
+ javascriptReturns?: {
+ type: string,
+ value: unknown
+ }[];
};
metadata: {
title?: string;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index ac6fabfd6..47322ef04 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -42,6 +42,36 @@ const successSchema = z.object({
})
.array()
.optional(),
+ actionResults: z.union([
+ z.object({
+ idx: z.number(),
+ type: z.literal("screenshot"),
+ result: z.object({
+ path: z.string(),
+ }),
+ }),
+ z.object({
+ idx: z.number(),
+ type: z.literal("scrape"),
+ result: z.union([
+ z.object({
+ url: z.string(),
+ html: z.string(),
+ }),
+ z.object({
+ url: z.string(),
+ accessibility: z.string(),
+ }),
+ ]),
+ }),
+ z.object({
+ idx: z.number(),
+ type: z.literal("executeJavascript"),
+ result: z.object({
+ return: z.string(),
+ }),
+ }),
+ ]).array().optional(),
// chrome-cdp only -- file download handler
file: z
@@ -138,7 +168,7 @@ export async function fireEngineCheckStatus(
} else if (
typeof status.error === "string" &&
// TODO: improve this later
- status.error.includes("Element")
+ (status.error.includes("Element") || status.error.includes("Javascript execution failed"))
) {
throw new ActionError(status.error.split("Error: ")[1]);
} else {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index c21d9f90e..93596fcee 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -274,6 +274,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
actions: {
screenshots: response.screenshots ?? [],
scrapes: response.actionContent ?? [],
+ javascriptReturns: (response.actionResults ?? []).filter(x => x.type === "executeJavascript").map(x => JSON.parse((x.result as any as { return: string }).return)),
},
}
: {}),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 0a688fc6a..ab2fe79ba 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -103,6 +103,10 @@ export type EngineScrapeResult = {
actions?: {
screenshots: string[];
scrapes: ScrapeActionContent[];
+ javascriptReturns: {
+ type: string;
+ value: unknown
+ }[];
};
};
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index 5cc9d1195..11fb8d747 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -141,6 +141,14 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
export interface ActionsResult {
screenshots: string[];
+ scrapes: ({
+ url: string;
+ html: string;
+ })[];
+ javascriptReturns: {
+ type: string;
+ value: unknown
+ }[];
}
/**