Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): fire-engine action metadata (#2699)
tom79a712bf5582
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f2546b589..e918df23f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -368,6 +368,12 @@ const actionSchema = z.union([export type Action = z.infer<typeof actionSchema>;+export type InternalAction = Action & {+ metadata?: {+ [key: string]: unknown;+ };+};+const actionsSchema = z.array(actionSchema).refine(actions => actions.length <= MAX_ACTIONS, {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex b18ffee60..340d85e79 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -29,7 +29,7 @@ import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";import { getInnerJson } from "@mendable/firecrawl-rs";import { hasFormatOfType } from "../../../../lib/format-utils";-import { Action } from "../../../../controllers/v1/types";+import { InternalAction } from "../../../../controllers/v1/types";import { AbortManagerThrownError } from "../../lib/abortManager";import { youtubePostprocessor } from "../../postprocessors/youtube";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";@@ -234,7 +234,7 @@ export async function scrapeURLWithFireEngineChromeCDP("engine.url": meta.url,"engine.team_id": meta.internalOptions.teamId,});- const actions: Action[] = [+ const actions: InternalAction[] = [// Transform waitFor option into an action (unsupported by chrome-cdp)...(meta.options.waitFor !== 0? [@@ -247,7 +247,10 @@ export async function scrapeURLWithFireEngineChromeCDP(: []),// Include specified actions- ...(meta.options.actions ?? []),+ ...(meta.options.actions ?? []).map(action => {+ const { metadata: _, ...rest } = action as InternalAction;+ return rest;+ }),// Transform screenshot format into an action (unsupported by chrome-cdp)...(hasFormatOfType(meta.options.formats, "screenshot")@@ -273,6 +276,7 @@ export async function scrapeURLWithFireEngineChromeCDP({type: "executeJavascript" as const,script: getBrandingScript(),+ metadata: { __firecrawl_internal: true },},]: []),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex eff8c8ca6..50d3f46ee 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -2,7 +2,7 @@ import { Logger } from "winston";import * as Sentry from "@sentry/node";import { z } from "zod";-import { Action } from "../../../../controllers/v1/types";+import { InternalAction } from "../../../../controllers/v1/types";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";import { getDocFromGCS } from "../../../../lib/gcs-jobs";@@ -50,7 +50,7 @@ export type FireEngineScrapeRequestCommon = {export type FireEngineScrapeRequestChromeCDP = {engine: "chrome-cdp";skipTlsVerification?: boolean;- actions?: Action[];+ actions?: InternalAction[];blockMedia?: boolean;mobile?: boolean;disableSmartWaitCache?: boolean;