Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): guard optional params when resolving origin in v1 methods (#4921)
Gergő Móriczfa74c805eaef
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 351406a27..fb9d7c5a2 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.42.3",+ "version": "4.42.4","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/optional-params.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/optional-params.test.tsnew file mode 100644index 000000000..d8caaca2e--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/optional-params.test.ts@@ -0,0 +1,70 @@+import { describe, test, expect, jest, beforeEach, afterEach } from "@jest/globals";+import axios from "axios";+import FirecrawlApp from "../../../v1";++const API_URL = "https://api.firecrawl.dev";++class FakeWebSocket {+ onopen: unknown = null;+ onmessage: unknown = null;+ onerror: unknown = null;+ onclose: unknown = null;+ close() {}+}++describe("v1 methods work without params", () => {+ let app: FirecrawlApp;+ let posts: Array<{ url: string; data: any }>;+ const originalWebSocket = (globalThis as any).WebSocket;++ beforeEach(() => {+ app = new FirecrawlApp({ apiKey: "fc-test", apiUrl: API_URL });+ posts = [];+ jest.spyOn(axios, "post").mockImplementation(async (url: string, data?: any) => {+ posts.push({ url, data });+ return { status: 200, data: { success: true, id: "job-1", data: [], links: [] } };+ });+ jest.spyOn(axios, "get").mockImplementation(async () => ({+ status: 200,+ data: { success: true, status: "completed", data: [{ markdown: "ok" }] },+ }));+ (globalThis as any).WebSocket = FakeWebSocket;+ });++ afterEach(() => {+ jest.restoreAllMocks();+ (globalThis as any).WebSocket = originalWebSocket;+ });++ const cases: Array<[string, string, (app: any) => Promise<unknown>]> = [+ ["scrapeUrl", "/v1/scrape", app => app.scrapeUrl("https://example.com")],+ ["search", "/v1/search", app => app.search("firecrawl")],+ ["crawlUrl", "/v1/crawl", app => app.crawlUrl("https://example.com")],+ ["asyncCrawlUrl", "/v1/crawl", app => app.asyncCrawlUrl("https://example.com")],+ ["crawlUrlAndWatch", "/v1/crawl", app => app.crawlUrlAndWatch("https://example.com")],+ ["mapUrl", "/v1/map", app => app.mapUrl("https://example.com")],+ ["batchScrapeUrls", "/v1/batch/scrape", app => app.batchScrapeUrls(["https://example.com"])],+ ["asyncBatchScrapeUrls", "/v1/batch/scrape", app => app.asyncBatchScrapeUrls(["https://example.com"])],+ ["batchScrapeUrlsAndWatch", "/v1/batch/scrape", app => app.batchScrapeUrlsAndWatch(["https://example.com"])],+ ["extract", "/v1/extract", app => app.extract(["https://example.com"])],+ ["asyncExtract", "/v1/extract", app => app.asyncExtract(["https://example.com"])],+ ["deepResearch", "/v1/deep-research", app => app.deepResearch("firecrawl")],+ ["asyncDeepResearch", "/v1/deep-research", app => app.asyncDeepResearch("firecrawl")],+ ["__deepResearch", "/v1/deep-research", app => app.__deepResearch("firecrawl")],+ ["__asyncDeepResearch", "/v1/deep-research", app => app.__asyncDeepResearch("firecrawl")],+ ["generateLLMsText", "/v1/llmstxt", app => app.generateLLMsText("https://example.com")],+ ["asyncGenerateLLMsText", "/v1/llmstxt", app => app.asyncGenerateLLMsText("https://example.com")],+ ];++ test.each(cases)("%s sends the default origin", async (_method, path, call) => {+ await expect(call(app)).resolves.toBeDefined();+ expect(posts).toHaveLength(1);+ expect(posts[0].url).toBe(`${API_URL}${path}`);+ expect(posts[0].data.origin).toBe(`js-sdk@${app.version}`);+ });++ test("an mcp origin passed in params is still forwarded", async () => {+ await app.batchScrapeUrls(["https://example.com"], { origin: "mcp-server" } as any);+ expect(posts[0].data.origin).toBe("mcp-server");+ });+});diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 55f1d749d..47bd7e9c5 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -726,7 +726,7 @@ export default class FirecrawlApp {"Content-Type": "application/json",Authorization: `Bearer ${this.apiKey}`,} as AxiosRequestHeaders;- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };if (jsonData?.extract?.schema) {jsonData = {...jsonData,@@ -793,7 +793,7 @@ export default class FirecrawlApp {lang: params?.lang ?? "en",country: params?.country ?? "us",location: params?.location,- origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,+ origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,timeout: params?.timeout ?? 60000,scrapeOptions: params?.scrapeOptions ?? { formats: [] },};@@ -857,7 +857,7 @@ export default class FirecrawlApp {idempotencyKey?: string): Promise<CrawlStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/crawl`,@@ -886,7 +886,7 @@ export default class FirecrawlApp {idempotencyKey?: string): Promise<CrawlResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/crawl`,@@ -1062,7 +1062,7 @@ export default class FirecrawlApp {*/async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };try {const response: AxiosResponse = await this.postRequest(@@ -1101,7 +1101,7 @@ export default class FirecrawlApp {maxConcurrency?: number,): Promise<BatchScrapeStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };if (jsonData?.extract?.schema) {jsonData = {...jsonData,@@ -1150,7 +1150,7 @@ export default class FirecrawlApp {ignoreInvalidURLs?: boolean,): Promise<BatchScrapeResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/batch/scrape`,@@ -1314,7 +1314,7 @@ export default class FirecrawlApp {try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },+ { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },headers);@@ -1374,7 +1374,7 @@ export default class FirecrawlApp {try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },+ { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },headers);@@ -1742,7 +1742,7 @@ export default class FirecrawlApp {*/async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { query, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { query, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };if (jsonData?.jsonOptions?.schema) {jsonData = {@@ -1890,7 +1890,7 @@ export default class FirecrawlApp {async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();try {- let jsonData: any = { topic, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { topic, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };const response: AxiosResponse = await this.postRequest(`${this.apiUrl}/v1/deep-research`,jsonData,@@ -2005,7 +2005,7 @@ export default class FirecrawlApp {*/async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: typeof (params as any).origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };try {const response: AxiosResponse = await this.postRequest(`${this.apiUrl}/v1/llmstxt`,