Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v1/map): timeout

Gergő Móricz57e98e83d7ab
diff --git a/apps/api/src/__tests__/snips/map.test.ts b/apps/api/src/__tests__/snips/map.test.ts
new file mode 100644
index 000000000..f2827fd42
--- /dev/null
+++ b/apps/api/src/__tests__/snips/map.test.ts
@@ -0,0 +1,42 @@
+import request from "supertest";
+import { configDotenv } from "dotenv";
+import { MapRequestInput } from "../../controllers/v1/types";
+
+configDotenv();
+const TEST_URL = "http://127.0.0.1:3002";
+
+async function map(body: MapRequestInput) {
+ return await request(TEST_URL)
+ .post("/v1/map")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send(body);
+}
+
+function expectMapToSucceed(response: Awaited<ReturnType<typeof map>>) {
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(Array.isArray(response.body.links)).toBe(true);
+ expect(response.body.links.length).toBeGreaterThan(0);
+}
+
+describe("Map tests", () => {
+ it("basic map succeeds", async () => {
+ const response = await map({
+ url: "http://firecrawl.dev",
+ });
+
+ expectMapToSucceed(response);
+ });
+
+ it("times out properly", async () => {
+ const response = await map({
+ url: "http://firecrawl.dev",
+ timeout: 1
+ });
+
+ expect(response.statusCode).toBe(408);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toBe("Request timed out");
+});
+});
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index c34f0220d..f42b57ea2 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -276,17 +276,34 @@ export async function mapController(
) {
req.body = mapRequestSchema.parse(req.body);
- const result = await getMapResults({
- url: req.body.url,
- search: req.body.search,
- limit: req.body.limit,
- ignoreSitemap: req.body.ignoreSitemap,
- includeSubdomains: req.body.includeSubdomains,
- crawlerOptions: req.body,
- origin: req.body.origin,
- teamId: req.auth.team_id,
- plan: req.auth.plan,
- });
+ let result: Awaited<ReturnType<typeof getMapResults>>;
+ try {
+ result = await Promise.race([
+ getMapResults({
+ url: req.body.url,
+ search: req.body.search,
+ limit: req.body.limit,
+ ignoreSitemap: req.body.ignoreSitemap,
+ includeSubdomains: req.body.includeSubdomains,
+ crawlerOptions: req.body,
+ origin: req.body.origin,
+ teamId: req.auth.team_id,
+ plan: req.auth.plan,
+ }),
+ ...(req.body.timeout !== undefined ? [
+ new Promise((resolve, reject) => setTimeout(() => reject("timeout"), req.body.timeout))
+ ] : []),
+ ]) as any;
+ } catch (error) {
+ if (error === "timeout") {
+ return res.status(408).json({
+ success: false,
+ error: "Request timed out",
+ });
+ } else {
+ throw error;
+ }
+ }
// Bill the team
billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 0f8d00e28..cb2526cee 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -429,6 +429,7 @@ export const mapRequestSchema = crawlerOptions
ignoreSitemap: z.boolean().default(false),
sitemapOnly: z.boolean().default(false),
limit: z.number().min(1).max(5000).default(5000),
+ timeout: z.number().positive().finite().optional(),
})
.strict(strictMessage);
@@ -438,6 +439,7 @@ export const mapRequestSchema = crawlerOptions
// };
export type MapRequest = z.infer<typeof mapRequestSchema>;
+export type MapRequestInput = z.input<typeof mapRequestSchema>;
export type Document = {
title?: string;
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index 58f772112..3fe7a8b9f 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -235,6 +235,7 @@ export interface MapParams {
includeSubdomains?: boolean;
sitemapOnly?: boolean;
limit?: number;
+ timeout?: number;
}
/**