Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): pdf action + housekeeping (#1702)
Gergő Móricze3948ae5b1b7
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex f8066a655..09b10c77d 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -677,7 +677,7 @@ describe("Scrape tests", () => {}, scrapeTimeout);});- describe("Screenshot (f-e/sb dependant)", () => {+ describe("Screenshot (f-e dependant)", () => {it.concurrent("screenshot format works", async () => {const response = await scrape({url: "http://firecrawl.dev",@@ -698,6 +698,21 @@ describe("Scrape tests", () => {expect(typeof response.screenshot).toBe("string");}, scrapeTimeout);});++ describe("PDF generation (f-e dependant)", () => {+ it.concurrent("works", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev",+ timeout: scrapeTimeout,+ actions: [{ type: "pdf" }],+ });++ expect(response.actions?.pdfs).toBeDefined();+ expect(response.actions?.pdfs?.length).toBe(1);+ expect(response.actions?.pdfs?.[0]).toBeDefined();+ expect(typeof response.actions?.pdfs?.[0]).toBe("string");+ }, scrapeTimeout);+ });describe("Proxy API (f-e dependant)", () => {it.concurrent("undefined works", async () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex e8ce8c51d..4682a9e9f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -170,55 +170,64 @@ function calculateTotalWaitTime(return waitFor + actionWaitTime;}-export const actionsSchema = z- .array(- z.union([- z- .object({- type: z.literal("wait"),- milliseconds: z.number().int().positive().finite().optional(),- selector: z.string().optional(),- })- .refine(- (data) =>- (data.milliseconds !== undefined || data.selector !== undefined) &&- !(data.milliseconds !== undefined && data.selector !== undefined),- {- message:- "Either 'milliseconds' or 'selector' must be provided, but not both.",- },- ),- z.object({- type: z.literal("click"),- selector: z.string(),- all: z.boolean().default(false),- }),- z.object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- }),- z.object({- type: z.literal("write"),- text: z.string(),- }),- z.object({- type: z.literal("press"),- key: z.string(),- }),- z.object({- type: z.literal("scroll"),- direction: z.enum(["up", "down"]).optional().default("down"),+export const actionSchema = z+ .union([+ z+ .object({+ type: z.literal("wait"),+ milliseconds: z.number().int().positive().finite().optional(),selector: z.string().optional(),- }),- z.object({- type: z.literal("scrape"),- }),- z.object({- type: z.literal("executeJavascript"),- script: z.string(),- }),- ]),- )+ })+ .refine(+ (data) =>+ (data.milliseconds !== undefined || data.selector !== undefined) &&+ !(data.milliseconds !== undefined && data.selector !== undefined),+ {+ message:+ "Either 'milliseconds' or 'selector' must be provided, but not both.",+ },+ ),+ z.object({+ type: z.literal("click"),+ selector: z.string(),+ all: z.boolean().default(false),+ }),+ z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ }),+ z.object({+ type: z.literal("write"),+ text: z.string(),+ }),+ z.object({+ type: z.literal("press"),+ key: z.string(),+ }),+ z.object({+ type: z.literal("scroll"),+ direction: z.enum(["up", "down"]).optional().default("down"),+ selector: z.string().optional(),+ }),+ z.object({+ type: z.literal("scrape"),+ }),+ z.object({+ type: z.literal("executeJavascript"),+ script: z.string(),+ }),+ z.object({+ type: z.literal("pdf"),+ landscape: z.boolean().default(false),+ scale: z.number().default(1),+ format: z.enum(['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'Letter', 'Legal', 'Tabloid', 'Ledger']).default("Letter"),+ }),+ ]);++export type Action = z.infer<typeof actionSchema>;++export const actionsSchema = z+ .array(actionSchema).refine((actions) => actions.length <= MAX_ACTIONS, {message: `Number of actions cannot exceed ${MAX_ACTIONS}`,})@@ -725,6 +734,7 @@ export type Document = {type: string;value: unknown;}[];+ pdfs?: string[];};changeTracking?: {previousScrapeAt: string | null;diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 42df545dc..30aa0430a 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -1,4 +1,4 @@-import type { Document as V1Document } from "../controllers/v1/types";+import type { Document as V1Document, Action } from "../controllers/v1/types";export interface Progress {current: number;@@ -12,42 +12,6 @@ export interface Progress {currentDocument?: Document;}-export type Action =- | {- type: "wait";- milliseconds?: number;- selector?: string;- }- | {- type: "click";- selector: string;- all?: boolean;- }- | {- type: "screenshot";- fullPage?: boolean;- }- | {- type: "write";- text: string;- }- | {- type: "press";- key: string;- }- | {- type: "scroll";- direction?: "up" | "down";- selector?: string;- }- | {- type: "scrape";- }- | {- type: "executeJavascript";- script: string;- };-export type PageOptions = {includeMarkdown?: boolean;includeExtract?: boolean;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 299380166..44222db76 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -73,6 +73,13 @@ const successSchema = z.object({return: z.string(),}),}),+ z.object({+ idx: z.number(),+ type: z.literal("pdf"),+ result: z.object({+ link: z.string(),+ }),+ }),]).array().optional(),// chrome-cdp only -- file download handlerdiff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 84884ec90..c10b8820f 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -22,12 +22,11 @@ import {UnsupportedFileError,} from "../../error";import * as Sentry from "@sentry/node";-import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";import { getInnerJSON } from "../../../../lib/html-transformer";-import { TimeoutSignal } from "../../../../controllers/v1/types";+import { Action, TimeoutSignal } from "../../../../controllers/v1/types";// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the@@ -283,6 +282,7 @@ export async function scrapeURLWithFireEngineChromeCDP(screenshots: response.screenshots ?? [],scrapes: response.actionContent ?? [],javascriptReturns: (response.actionResults ?? []).filter(x => x.type === "executeJavascript").map(x => JSON.parse((x.result as any as { return: string }).return)),+ pdfs: (response.actionResults ?? []).filter(x => x.type === "pdf").map(x => JSON.parse((x.result as any as { link: string }).link)),},}: {}),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 5261b697a..accee57e9 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -2,7 +2,7 @@ import { Logger } from "winston";import * as Sentry from "@sentry/node";import { z } from "zod";-import { Action } from "../../../../lib/entities";+import { Action } from "../../../../controllers/v1/types";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 713e1959c..5b254259d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -109,6 +109,7 @@ export type EngineScrapeResult = {type: string;value: unknown}[];+ pdfs: string[];};numPages?: number;