Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): fire-engine action metadata (#2699)

tom79a712bf5582
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index f2546b589..e918df23f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -368,6 +368,12 @@ const actionSchema = z.union([
export type Action = z.infer<typeof actionSchema>;
+export type InternalAction = Action & {
+ metadata?: {
+ [key: string]: unknown;
+ };
+};
+
const actionsSchema = z
.array(actionSchema)
.refine(actions => actions.length <= MAX_ACTIONS, {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index b18ffee60..340d85e79 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -29,7 +29,7 @@ import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
import { getInnerJson } from "@mendable/firecrawl-rs";
import { hasFormatOfType } from "../../../../lib/format-utils";
-import { Action } from "../../../../controllers/v1/types";
+import { InternalAction } from "../../../../controllers/v1/types";
import { AbortManagerThrownError } from "../../lib/abortManager";
import { youtubePostprocessor } from "../../postprocessors/youtube";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
@@ -234,7 +234,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
"engine.url": meta.url,
"engine.team_id": meta.internalOptions.teamId,
});
- const actions: Action[] = [
+ const actions: InternalAction[] = [
// Transform waitFor option into an action (unsupported by chrome-cdp)
...(meta.options.waitFor !== 0
? [
@@ -247,7 +247,10 @@ export async function scrapeURLWithFireEngineChromeCDP(
: []),
// Include specified actions
- ...(meta.options.actions ?? []),
+ ...(meta.options.actions ?? []).map(action => {
+ const { metadata: _, ...rest } = action as InternalAction;
+ return rest;
+ }),
// Transform screenshot format into an action (unsupported by chrome-cdp)
...(hasFormatOfType(meta.options.formats, "screenshot")
@@ -273,6 +276,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
{
type: "executeJavascript" as const,
script: getBrandingScript(),
+ metadata: { __firecrawl_internal: true },
},
]
: []),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index eff8c8ca6..50d3f46ee 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -2,7 +2,7 @@ import { Logger } from "winston";
import * as Sentry from "@sentry/node";
import { z } from "zod";
-import { Action } from "../../../../controllers/v1/types";
+import { InternalAction } from "../../../../controllers/v1/types";
import { robustFetch } from "../../lib/fetch";
import { MockState } from "../../lib/mock";
import { getDocFromGCS } from "../../../../lib/gcs-jobs";
@@ -50,7 +50,7 @@ export type FireEngineScrapeRequestCommon = {
export type FireEngineScrapeRequestChromeCDP = {
engine: "chrome-cdp";
skipTlsVerification?: boolean;
- actions?: Action[];
+ actions?: InternalAction[];
blockMedia?: boolean;
mobile?: boolean;
disableSmartWaitCache?: boolean;