Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): pdf action + housekeeping (#1702)

Gergő Móricze3948ae5b1b7
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index f8066a655..09b10c77d 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -677,7 +677,7 @@ describe("Scrape tests", () => {
}, scrapeTimeout);
});
- describe("Screenshot (f-e/sb dependant)", () => {
+ describe("Screenshot (f-e dependant)", () => {
it.concurrent("screenshot format works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
@@ -698,6 +698,21 @@ describe("Scrape tests", () => {
expect(typeof response.screenshot).toBe("string");
}, scrapeTimeout);
});
+
+ describe("PDF generation (f-e dependant)", () => {
+ it.concurrent("works", async () => {
+ const response = await scrape({
+ url: "https://firecrawl.dev",
+ timeout: scrapeTimeout,
+ actions: [{ type: "pdf" }],
+ });
+
+ expect(response.actions?.pdfs).toBeDefined();
+ expect(response.actions?.pdfs?.length).toBe(1);
+ expect(response.actions?.pdfs?.[0]).toBeDefined();
+ expect(typeof response.actions?.pdfs?.[0]).toBe("string");
+ }, scrapeTimeout);
+ });
describe("Proxy API (f-e dependant)", () => {
it.concurrent("undefined works", async () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index e8ce8c51d..4682a9e9f 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -170,55 +170,64 @@ function calculateTotalWaitTime(
return waitFor + actionWaitTime;
}
-export const actionsSchema = z
- .array(
- z.union([
- z
- .object({
- type: z.literal("wait"),
- milliseconds: z.number().int().positive().finite().optional(),
- selector: z.string().optional(),
- })
- .refine(
- (data) =>
- (data.milliseconds !== undefined || data.selector !== undefined) &&
- !(data.milliseconds !== undefined && data.selector !== undefined),
- {
- message:
- "Either 'milliseconds' or 'selector' must be provided, but not both.",
- },
- ),
- z.object({
- type: z.literal("click"),
- selector: z.string(),
- all: z.boolean().default(false),
- }),
- z.object({
- type: z.literal("screenshot"),
- fullPage: z.boolean().default(false),
- }),
- z.object({
- type: z.literal("write"),
- text: z.string(),
- }),
- z.object({
- type: z.literal("press"),
- key: z.string(),
- }),
- z.object({
- type: z.literal("scroll"),
- direction: z.enum(["up", "down"]).optional().default("down"),
+export const actionSchema = z
+ .union([
+ z
+ .object({
+ type: z.literal("wait"),
+ milliseconds: z.number().int().positive().finite().optional(),
selector: z.string().optional(),
- }),
- z.object({
- type: z.literal("scrape"),
- }),
- z.object({
- type: z.literal("executeJavascript"),
- script: z.string(),
- }),
- ]),
- )
+ })
+ .refine(
+ (data) =>
+ (data.milliseconds !== undefined || data.selector !== undefined) &&
+ !(data.milliseconds !== undefined && data.selector !== undefined),
+ {
+ message:
+ "Either 'milliseconds' or 'selector' must be provided, but not both.",
+ },
+ ),
+ z.object({
+ type: z.literal("click"),
+ selector: z.string(),
+ all: z.boolean().default(false),
+ }),
+ z.object({
+ type: z.literal("screenshot"),
+ fullPage: z.boolean().default(false),
+ }),
+ z.object({
+ type: z.literal("write"),
+ text: z.string(),
+ }),
+ z.object({
+ type: z.literal("press"),
+ key: z.string(),
+ }),
+ z.object({
+ type: z.literal("scroll"),
+ direction: z.enum(["up", "down"]).optional().default("down"),
+ selector: z.string().optional(),
+ }),
+ z.object({
+ type: z.literal("scrape"),
+ }),
+ z.object({
+ type: z.literal("executeJavascript"),
+ script: z.string(),
+ }),
+ z.object({
+ type: z.literal("pdf"),
+ landscape: z.boolean().default(false),
+ scale: z.number().default(1),
+ format: z.enum(['A0', 'A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'Letter', 'Legal', 'Tabloid', 'Ledger']).default("Letter"),
+ }),
+ ]);
+
+export type Action = z.infer<typeof actionSchema>;
+
+export const actionsSchema = z
+ .array(actionSchema)
.refine((actions) => actions.length <= MAX_ACTIONS, {
message: `Number of actions cannot exceed ${MAX_ACTIONS}`,
})
@@ -725,6 +734,7 @@ export type Document = {
type: string;
value: unknown;
}[];
+ pdfs?: string[];
};
changeTracking?: {
previousScrapeAt: string | null;
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 42df545dc..30aa0430a 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -1,4 +1,4 @@
-import type { Document as V1Document } from "../controllers/v1/types";
+import type { Document as V1Document, Action } from "../controllers/v1/types";
export interface Progress {
current: number;
@@ -12,42 +12,6 @@ export interface Progress {
currentDocument?: Document;
}
-export type Action =
- | {
- type: "wait";
- milliseconds?: number;
- selector?: string;
- }
- | {
- type: "click";
- selector: string;
- all?: boolean;
- }
- | {
- type: "screenshot";
- fullPage?: boolean;
- }
- | {
- type: "write";
- text: string;
- }
- | {
- type: "press";
- key: string;
- }
- | {
- type: "scroll";
- direction?: "up" | "down";
- selector?: string;
- }
- | {
- type: "scrape";
- }
- | {
- type: "executeJavascript";
- script: string;
- };
-
export type PageOptions = {
includeMarkdown?: boolean;
includeExtract?: boolean;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 299380166..44222db76 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -73,6 +73,13 @@ const successSchema = z.object({
return: z.string(),
}),
}),
+ z.object({
+ idx: z.number(),
+ type: z.literal("pdf"),
+ result: z.object({
+ link: z.string(),
+ }),
+ }),
]).array().optional(),
// chrome-cdp only -- file download handler
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 84884ec90..c10b8820f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -22,12 +22,11 @@ import {
UnsupportedFileError,
} from "../../error";
import * as Sentry from "@sentry/node";
-import { Action } from "../../../../lib/entities";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
import { getInnerJSON } from "../../../../lib/html-transformer";
-import { TimeoutSignal } from "../../../../controllers/v1/types";
+import { Action, TimeoutSignal } from "../../../../controllers/v1/types";
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
@@ -283,6 +282,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
screenshots: response.screenshots ?? [],
scrapes: response.actionContent ?? [],
javascriptReturns: (response.actionResults ?? []).filter(x => x.type === "executeJavascript").map(x => JSON.parse((x.result as any as { return: string }).return)),
+ pdfs: (response.actionResults ?? []).filter(x => x.type === "pdf").map(x => JSON.parse((x.result as any as { link: string }).link)),
},
}
: {}),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 5261b697a..accee57e9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -2,7 +2,7 @@ import { Logger } from "winston";
import * as Sentry from "@sentry/node";
import { z } from "zod";
-import { Action } from "../../../../lib/entities";
+import { Action } from "../../../../controllers/v1/types";
import { robustFetch } from "../../lib/fetch";
import { MockState } from "../../lib/mock";
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 713e1959c..5b254259d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -109,6 +109,7 @@ export type EngineScrapeResult = {
type: string;
value: unknown
}[];
+ pdfs: string[];
};
numPages?: number;