Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1/map): timeout
Gergő Móricz57e98e83d7ab
diff --git a/apps/api/src/__tests__/snips/map.test.ts b/apps/api/src/__tests__/snips/map.test.tsnew file mode 100644index 000000000..f2827fd42--- /dev/null+++ b/apps/api/src/__tests__/snips/map.test.ts@@ -0,0 +1,42 @@+import request from "supertest";+import { configDotenv } from "dotenv";+import { MapRequestInput } from "../../controllers/v1/types";++configDotenv();+const TEST_URL = "http://127.0.0.1:3002";++async function map(body: MapRequestInput) {+ return await request(TEST_URL)+ .post("/v1/map")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send(body);+}++function expectMapToSucceed(response: Awaited<ReturnType<typeof map>>) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(Array.isArray(response.body.links)).toBe(true);+ expect(response.body.links.length).toBeGreaterThan(0);+}++describe("Map tests", () => {+ it("basic map succeeds", async () => {+ const response = await map({+ url: "http://firecrawl.dev",+ });++ expectMapToSucceed(response);+ });++ it("times out properly", async () => {+ const response = await map({+ url: "http://firecrawl.dev",+ timeout: 1+ });++ expect(response.statusCode).toBe(408);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toBe("Request timed out");+});+});diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex c34f0220d..f42b57ea2 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -276,17 +276,34 @@ export async function mapController() {req.body = mapRequestSchema.parse(req.body);- const result = await getMapResults({- url: req.body.url,- search: req.body.search,- limit: req.body.limit,- ignoreSitemap: req.body.ignoreSitemap,- includeSubdomains: req.body.includeSubdomains,- crawlerOptions: req.body,- origin: req.body.origin,- teamId: req.auth.team_id,- plan: req.auth.plan,- });+ let result: Awaited<ReturnType<typeof getMapResults>>;+ try {+ result = await Promise.race([+ getMapResults({+ url: req.body.url,+ search: req.body.search,+ limit: req.body.limit,+ ignoreSitemap: req.body.ignoreSitemap,+ includeSubdomains: req.body.includeSubdomains,+ crawlerOptions: req.body,+ origin: req.body.origin,+ teamId: req.auth.team_id,+ plan: req.auth.plan,+ }),+ ...(req.body.timeout !== undefined ? [+ new Promise((resolve, reject) => setTimeout(() => reject("timeout"), req.body.timeout))+ ] : []),+ ]) as any;+ } catch (error) {+ if (error === "timeout") {+ return res.status(408).json({+ success: false,+ error: "Request timed out",+ });+ } else {+ throw error;+ }+ }// Bill the teambillTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 0f8d00e28..cb2526cee 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -429,6 +429,7 @@ export const mapRequestSchema = crawlerOptionsignoreSitemap: z.boolean().default(false),sitemapOnly: z.boolean().default(false),limit: z.number().min(1).max(5000).default(5000),+ timeout: z.number().positive().finite().optional(),}).strict(strictMessage);@@ -438,6 +439,7 @@ export const mapRequestSchema = crawlerOptions// };export type MapRequest = z.infer<typeof mapRequestSchema>;+export type MapRequestInput = z.input<typeof mapRequestSchema>;export type Document = {title?: string;diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 58f772112..3fe7a8b9f 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -235,6 +235,7 @@ export interface MapParams {includeSubdomains?: boolean;sitemapOnly?: boolean;limit?: number;+ timeout?: number;}/**