Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdks): let wrapper libraries set and default the request origin (#5217)
Erik Engervall700f7d488dae
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 8bf7bb8d7..c45e89c50 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -33,6 +33,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";+import { isSdkOrigin } from "../../lib/sdk-origin";async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,@@ -291,8 +292,7 @@ export async function extractController((await getTeamIdSyncB(req.auth.team_id)) &&req.body.origin !== "api-sdk" &&req.body.origin !== "website" &&- !req.body.origin.startsWith("python-sdk@") &&- !req.body.origin.startsWith("js-sdk@")+ !isSdkOrigin(req.body.origin)) {return await oldExtract(req, res, extractId);}diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.tsindex f0a6979ee..8465f3c49 100644--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts@@ -230,6 +230,8 @@ describe("agentStatusController", () => {["[email protected]", "spark-1-pro"],// everything else sees the real model["[email protected]", "spark-2"],+ ["[email protected]", "spark-1-pro"],+ ["[email protected]", "spark-2"],["[email protected]", "spark-2"],["[email protected]", "spark-2"],["[email protected]", "spark-2"],diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.tsindex b260ab77b..f91ff6551 100644--- a/apps/api/src/controllers/v2/agent-status.ts+++ b/apps/api/src/controllers/v2/agent-status.ts@@ -22,7 +22,7 @@ import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";// version regex is end-anchored so a prerelease of the fix (e.g.// "4.37.1rc0", which may predate the Literal widening) also fails to// parse and gets lied to rather than crashed.-const PYTHON_SDK_ORIGIN = /^python-sdk@(.+)$/;+const PYTHON_SDK_ORIGIN = /^python-sdk(?:\.[^@]*)?@(.+)$/;const PYTHON_SDK_SPARK_2_FIX = [4, 37, 1];function isIncompatiblePythonSdkOrigin(origin: unknown): boolean {diff --git a/apps/api/src/lib/sdk-origin.test.ts b/apps/api/src/lib/sdk-origin.test.tsnew file mode 100644index 000000000..59f3506f4--- /dev/null+++ b/apps/api/src/lib/sdk-origin.test.ts@@ -0,0 +1,24 @@+import { describe, expect, it } from "vitest";+import { isSdkOrigin } from "./sdk-origin";++describe("isSdkOrigin", () => {+ it.each([+ "[email protected]",+ "[email protected]",+ "[email protected]",+ "[email protected]",+ ])("accepts %s", origin => {+ expect(isSdkOrigin(origin)).toBe(true);+ });++ it.each([+ "api",+ "website",+ "mcp-server",+ "[email protected]",+ "[email protected]",+ "[email protected]",+ ])("rejects %s", origin => {+ expect(isSdkOrigin(origin)).toBe(false);+ });+});diff --git a/apps/api/src/lib/sdk-origin.ts b/apps/api/src/lib/sdk-origin.tsnew file mode 100644index 000000000..dd3fac7fc--- /dev/null+++ b/apps/api/src/lib/sdk-origin.ts@@ -0,0 +1,3 @@+export function isSdkOrigin(origin: string): boolean {+ return /^(python|js)-sdk[@.]/.test(origin);+}diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex d97928399..c5c518215 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -37,6 +37,16 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {console.log(crawlResponse);```+### Request origin++Requests carry an `origin` tag that identifies the client. Pass the `origin` option to set it. Set `origin` on a single call to override it for that call, except for `app.research` methods, which always use the client origin.++```js+const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY', origin: '[email protected]' });+```++By default the tag is `js-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `js-sdk.<package>@<version>` instead, for example `js-sdk.scope-name@<version>`. Application code, REPLs and bundled files without `node_modules` in their path keep the plain default. Detection runs once when the client is created and sends only the package name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.+### Scraping a URLTo scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.tsindex 6dee3ed32..0690e569c 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts@@ -14,6 +14,7 @@ const hints = [];function httpFor(body: object) {return {+ getOrigin: () => "js-sdk@test",post: vi.fn<any>().mockResolvedValue({ status: 200, data: body }),postMultipart: vi.fn<any>()diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.tsnew file mode 100644index 000000000..6d2901a89--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.ts@@ -0,0 +1,132 @@+import { afterEach, beforeEach, describe, test, expect } from "vitest";+import type { AxiosAdapter } from "axios";+import { FirecrawlClient } from "../../../v2/client";+import { Firecrawl } from "../../../index";+import { getVersion } from "../../../v2/utils/getVersion";+import { detectCallerPackage } from "../../../utils/callerOrigin";++function withCapture<T extends { http: any }>(client: T) {+ const bodies: Array<Record<string, unknown>> = [];+ const adapter: AxiosAdapter = async config => {+ bodies.push(JSON.parse(config.data));+ return {+ data: { success: true, data: { markdown: "# hello" } },+ status: 200,+ statusText: "OK",+ headers: {},+ config,+ };+ };+ client.http.instance.defaults.adapter = adapter;+ return { client, bodies };+}++describe("v2 request origin", () => {+ const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;+ afterEach(() => {+ if (previousOptOut === undefined) {+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;+ } else {+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;+ }+ });++ test("defaults to the plain SDK origin when application code creates the client", async () => {+ const { client, bodies } = withCapture(new FirecrawlClient({ apiKey: "fc-test" }));+ await client.scrape("https://example.com");+ expect(bodies[0]!.origin).toBe(`js-sdk@${getVersion()}`);+ });++ test("the origin client option is stamped into request bodies", async () => {+ const { client, bodies } = withCapture(+ new FirecrawlClient({ apiKey: "fc-test", origin: "[email protected]" }),+ );+ await client.scrape("https://example.com");+ expect(bodies[0]!.origin).toBe("[email protected]");+ });++ test("a per-call origin wins over the client origin", async () => {+ const { client, bodies } = withCapture(+ new FirecrawlClient({ apiKey: "fc-test", origin: "[email protected]" }),+ );+ await client.scrape("https://example.com", { origin: "per-call" } as any);+ await client.scrape("https://example.com", { origin: "mcp-server" } as any);+ expect(bodies.map(b => b.origin)).toEqual(["per-call", "mcp-server"]);+ });++ test("the unified client forwards the origin option to v1", () => {+ const app = new Firecrawl({ apiKey: "fc-test", origin: "[email protected]" });+ expect((app.v1 as any).clientOrigin).toBe("[email protected]");+ });++});++describe("detectCallerPackage", () => {+ const sdkFrames = [+ "Error",+ " at detectCallerPackage (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:10:5)",+ " at new FirecrawlClient (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:20:9)",+ ];+ const stackFrom = (...callers: string[]) =>+ [...sdkFrames, ...callers.map(c => ` at ${c}`)].join("\n");+ const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;++ beforeEach(() => {+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;+ });++ afterEach(() => {+ if (previousOptOut === undefined) {+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;+ } else {+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;+ }+ });++ test("names the scoped package that created the client", () => {+ const stack = stackFrom(+ "make (file:///srv/app/node_modules/@Scope/Some_Agent/dist/index.js:3:12)",+ "main (file:///srv/app/src/main.js:1:1)",+ );+ expect(detectCallerPackage(stack)).toBe("scope-some-agent");+ });++ test("uses the last node_modules segment of a nested install", () => {+ const stack = stackFrom(+ "make (/srv/app/node_modules/.pnpm/[email protected]/node_modules/inner-pkg/lib/index.cjs:3:12)",+ );+ expect(detectCallerPackage(stack)).toBe("inner-pkg");+ });++ test("handles windows paths with parentheses and anonymous frames", () => {+ const stack = stackFrom(+ "Array.map (<anonymous>)",+ "C:\\Program Files (x86)\\app\\node_modules\\wrapper\\index.js:4:2",+ );+ expect(detectCallerPackage(stack)).toBe("wrapper");+ });++ test("returns null when application code created the client", () => {+ const stack = stackFrom(+ "main (file:///srv/app/src/main.js:1:1)",+ "make (file:///srv/app/node_modules/wrapper/index.js:3:12)",+ );+ expect(detectCallerPackage(stack)).toBeNull();+ });++ test("returns null for a package whose slug would read as another traffic class", () => {+ const stack = stackFrom("make (/srv/app/node_modules/my-playground-kit/index.js:3:12)");+ expect(detectCallerPackage(stack)).toBeNull();+ });++ test("returns null for a bundle without node_modules in its path", () => {+ expect(detectCallerPackage(stackFrom("make (/srv/app/dist/bundle.js:1:1)"))).toBeNull();+ });++ test("FIRECRAWL_DISABLE_CALLER_ORIGIN turns detection off", () => {+ const stack = stackFrom("make (/srv/app/node_modules/wrapper/index.js:3:12)");+ expect(detectCallerPackage(stack)).toBe("wrapper");+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = "1";+ expect(detectCallerPackage(stack)).toBeNull();+ });+});diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.tsindex 088e93d22..5dbc7cbe1 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts@@ -12,6 +12,7 @@ function makeClient() {const calls: string[] = [];const http = {+ getOrigin: () => "js-sdk@test",get: async (url: string) => {calls.push(url);return responder(url);diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex a52be2453..0250f0826 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -37,6 +37,7 @@ export class Firecrawl extends V2 {this._v1Opts = {apiKey: resolved.apiKey,apiUrl: resolved.apiUrl,+ origin: this.http.getOrigin(),};}diff --git a/apps/js-sdk/firecrawl/src/utils/callerOrigin.ts b/apps/js-sdk/firecrawl/src/utils/callerOrigin.tsnew file mode 100644index 000000000..b32d35f8d--- /dev/null+++ b/apps/js-sdk/firecrawl/src/utils/callerOrigin.ts@@ -0,0 +1,74 @@+const NODE_MODULES = "/node_modules/";++function packageOf(file: string): string | null {+ let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");+ try {+ path = decodeURIComponent(path);+ } catch {}+ const at = path.lastIndexOf(NODE_MODULES);+ if (at < 0) return null;+ const [first, second] = path.slice(at + NODE_MODULES.length).split("/");+ const name = first?.startsWith("@") ? (second ? `${first}/${second}` : "") : first;+ return name && !name.startsWith(".") ? name : null;+}++/** File of a V8 stack line, or null for frames without one (native, anonymous, runtime internals). */+function frameFile(line: string): string | null {+ const text = line.trim().replace(/^at /, "");+ const open = text.indexOf(" (");+ const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;+ const match = /^(.*):\d+:\d+$/.exec(location);+ return match && !match[1]!.startsWith("node:") ? match[1]! : null;+}++// Downstream surface classification matches "playground" anywhere in an origin.+function slugify(name: string): string {+ return name+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, "-")+ .replace(/^-+|-+$/g, "")+ .slice(0, 48)+ .replace(/-+$/, "");+}++/**+ * Slug of the installed package that created the client, or null when the+ * caller is application code or detection is disabled. Never throws.+ */+export function detectCallerPackage(stack?: string): string | null {+ try {+ if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;+ if (stack === undefined) {+ const limit = Error.stackTraceLimit;+ Error.stackTraceLimit = 50;+ try {+ stack = new Error().stack ?? "";+ } finally {+ Error.stackTraceLimit = limit;+ }+ }+ let ownFile: string | null = null;+ let ownPackage: string | null = null;+ for (const line of stack.split("\n")) {+ if (!/^\s*at /.test(line)) continue;+ const file = frameFile(line);+ if (!file) continue;+ if (ownFile === null) {+ ownFile = file;+ ownPackage = packageOf(file);+ }+ if (file === ownFile) continue;+ const name = packageOf(file);+ if (name && name === ownPackage) continue;+ const slug = name ? slugify(name) : "";+ return slug && !slug.includes("playground") ? slug : null;+ }+ } catch {}+ return null;+}++/** Origin sent with requests: `js-sdk@<version>`, or `js-sdk.<package>@<version>` for wrapper libraries. */+export function defaultOrigin(version: string): string {+ const caller = detectCallerPackage();+ return caller ? `js-sdk.${caller}@${version}` : `js-sdk@${version}`;+}diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex caba47551..2661a9fb8 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -4,15 +4,18 @@ import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";import { TypedEventTarget } from "typescript-event-target";import type { RequiresAction } from "../v2/types";import { pinToApiOrigin } from "../utils/apiOrigin";+import { detectCallerPackage } from "../utils/callerOrigin";/*** Configuration interface for FirecrawlApp.* @param apiKey - Optional API key for authentication.* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.+ * @param origin - Optional attribution tag stamped into request bodies.*/export interface FirecrawlAppConfig {apiKey?: string | null;apiUrl?: string | null;+ origin?: string;}/**@@ -666,6 +669,8 @@ export default class FirecrawlApp {public apiKey: string;public apiUrl: string;public version: string = "1.25.1";+ private readonly clientOrigin?: string;+ private readonly callerPackage: string | null;private isCloudService(url: string): boolean {return url.includes('api.firecrawl.dev');@@ -698,7 +703,7 @@ export default class FirecrawlApp {* Initializes a new instance of the FirecrawlApp class.* @param config - Configuration options for the FirecrawlApp instance.*/- constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {+ constructor({ apiKey = null, apiUrl = null, origin }: FirecrawlAppConfig) {const baseUrl = apiUrl || "https://api.firecrawl.dev";if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {@@ -707,9 +712,19 @@ export default class FirecrawlApp {this.apiKey = apiKey || '';this.apiUrl = baseUrl;+ this.clientOrigin = origin || undefined;+ this.callerPackage = this.clientOrigin ? null : detectCallerPackage();this.init();}+ private resolveOrigin(params?: any): string {+ if (typeof params?.origin === "string" && params.origin) return params.origin;+ if (this.clientOrigin) return this.clientOrigin;+ return this.callerPackage+ ? `js-sdk.${this.callerPackage}@${this.version}`+ : `js-sdk@${this.version}`;+ }+/*** Scrapes a URL using the Firecrawl API.* @param url - The URL to scrape.@@ -724,7 +739,7 @@ export default class FirecrawlApp {"Content-Type": "application/json",Authorization: `Bearer ${this.apiKey}`,} as AxiosRequestHeaders;- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };if (jsonData?.extract?.schema) {jsonData = {...jsonData,@@ -791,7 +806,7 @@ export default class FirecrawlApp {lang: params?.lang ?? "en",country: params?.country ?? "us",location: params?.location,- origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,+ origin: this.resolveOrigin(params),timeout: params?.timeout ?? 60000,scrapeOptions: params?.scrapeOptions ?? { formats: [] },};@@ -855,7 +870,7 @@ export default class FirecrawlApp {idempotencyKey?: string): Promise<CrawlStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/crawl`,@@ -884,7 +899,7 @@ export default class FirecrawlApp {idempotencyKey?: string): Promise<CrawlResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/crawl`,@@ -1060,7 +1075,7 @@ export default class FirecrawlApp {*/async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };try {const response: AxiosResponse = await this.postRequest(@@ -1099,7 +1114,7 @@ export default class FirecrawlApp {maxConcurrency?: number,): Promise<BatchScrapeStatusResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };if (jsonData?.extract?.schema) {jsonData = {...jsonData,@@ -1148,7 +1163,7 @@ export default class FirecrawlApp {ignoreInvalidURLs?: boolean,): Promise<BatchScrapeResponse | ErrorResponse> {const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/batch/scrape`,@@ -1312,7 +1327,7 @@ export default class FirecrawlApp {try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },+ { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },headers);@@ -1372,7 +1387,7 @@ export default class FirecrawlApp {try {const response: AxiosResponse = await this.postRequest(this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },+ { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },headers);@@ -1740,7 +1755,7 @@ export default class FirecrawlApp {*/async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { query, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { query, ...params, origin: this.resolveOrigin(params) };if (jsonData?.jsonOptions?.schema) {jsonData = {@@ -1888,7 +1903,7 @@ export default class FirecrawlApp {async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();try {- let jsonData: any = { topic, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { topic, ...params, origin: this.resolveOrigin(params) };const response: AxiosResponse = await this.postRequest(`${this.apiUrl}/v1/deep-research`,jsonData,@@ -2003,7 +2018,7 @@ export default class FirecrawlApp {*/async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };try {const response: AxiosResponse = await this.postRequest(`${this.apiUrl}/v1/llmstxt`,diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 41d3f1da9..b9be6b349 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -101,6 +101,8 @@ import type {GetMonitorCheckOptions,UpdateMonitorRequest,} from "./types";+import { defaultOrigin } from "../utils/callerOrigin";+import { getVersion } from "./utils/getVersion";import { Watcher } from "./watcher";import type { WatcherOptions } from "./watcher";import * as zt from "zod";@@ -130,6 +132,12 @@ export interface FirecrawlClientOptions {maxRetries?: number;/** Exponential backoff factor for retries (optional). */backoffFactor?: number;+ /**+ * Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,+ * or `js-sdk.<package>@<version>` when an installed package creates the client+ * (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).+ */+ origin?: string;}/** Accepts a plain API key string or a full options object. */@@ -140,7 +148,7 @@ export type FirecrawlClientInput = FirecrawlClientOptions | string;*/export class FirecrawlClient {- private readonly http: HttpClient;+ protected readonly http: HttpClient;private _research?: ResearchClient;private isCloudService(url: string): boolean {@@ -168,6 +176,7 @@ export class FirecrawlClient {timeoutMs: opts.timeoutMs,maxRetries: opts.maxRetries,backoffFactor: opts.backoffFactor,+ origin: opts.origin || defaultOrigin(getVersion()),});}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts b/apps/js-sdk/firecrawl/src/v2/methods/parse.tsindex 183ca6730..5436c3e32 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts@@ -6,7 +6,6 @@ import {throwForBadResponse,normalizeAxiosError,} from "../utils/errorHandler";-import { getVersion } from "../utils/getVersion";function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {if (typeof Blob !== "undefined" && input instanceof Blob) {@@ -57,13 +56,9 @@ export async function parse(if (options) ensureValidParseOptions(options);- const version = getVersion();const normalizedOptions: ParseOptions = {...(options ?? {}),- origin:- typeof options?.origin === "string" && options.origin.includes("mcp")- ? options.origin- : (options?.origin ?? `js-sdk@${version}`),+ origin: options?.origin || http.getOrigin(),};const formData = new FormData();diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.tsindex 5292b2997..270336bb2 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts@@ -12,7 +12,6 @@ import type {import { SdkError } from "../types";import { HttpClient } from "../utils/httpClient";import { throwForBadResponse } from "../utils/errorHandler";-import { getVersion } from "../utils/getVersion";const BASE = "/v2/search/research";@@ -32,10 +31,14 @@ function appendParam(}}-function withQuery(path: string, params: URLSearchParams): string {+function withQuery(+ origin: string,+ path: string,+ params: URLSearchParams,+): string {// Research endpoints are GETs, so the POST-body origin injection in// HttpClient never applies — attach it as a query param instead.- params.append("origin", `js-sdk@${getVersion()}`);+ params.append("origin", origin);const qs = params.toString();return qs ? `${path}?${qs}` : path;}@@ -121,7 +124,7 @@ export class ResearchClient {appendParam(params, "to", options.to);try {const res = await this.http.get<SearchPapersResponse>(- withQuery(`${BASE}/papers`, params),+ withQuery(this.http.getOrigin(), `${BASE}/papers`, params),);if (res.status !== 200) throwForBadResponse(res, "search papers");return res.data;@@ -161,7 +164,13 @@ export class ResearchClient {try {const res = await this.http.get<PaperMetadataResponse | ReadPaperResponse- >(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));+ >(+ withQuery(+ this.http.getOrigin(),+ `${BASE}/papers/${encodeURIComponent(id)}`,+ params,+ ),+ );if (res.status !== 200) throwForBadResponse(res, "get paper");return res.data;} catch (err) {@@ -191,7 +200,11 @@ export class ResearchClient {appendParam(params, "anchor", options.anchor);try {const res = await this.http.get<SimilarPapersResponse>(- withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params),+ withQuery(+ this.http.getOrigin(),+ `${BASE}/papers/${encodeURIComponent(id)}/similar`,+ params,+ ),);if (res.status !== 200) throwForBadResponse(res, "find similar papers");return res.data;@@ -226,7 +239,7 @@ export class ResearchClient {appendParam(params, "k", options.k);try {const res = await this.http.get<GitHubSearchResponse>(- withQuery(`${BASE}/github`, params),+ withQuery(this.http.getOrigin(), `${BASE}/github`, params),);if (res.status !== 200) throwForBadResponse(res, "search github");return res.data;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsindex b59c047a8..495565e85 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -12,6 +12,7 @@ export interface HttpClientOptions {timeoutMs?: number;maxRetries?: number;backoffFactor?: number; // seconds factor for 0.5, 1, 2...+ origin?: string;}export interface RequestOptions {@@ -25,12 +26,14 @@ export class HttpClient {private readonly apiUrl: string;private readonly maxRetries: number;private readonly backoffFactor: number;+ private readonly origin: string;constructor(options: HttpClientOptions) {this.apiKey = options.apiKey;this.apiUrl = options.apiUrl.replace(/\/$/, "");this.maxRetries = options.maxRetries ?? 3;this.backoffFactor = options.backoffFactor ?? 0.5;+ this.origin = options.origin || `js-sdk@${getVersion()}`;this.instance = axios.create({baseURL: this.apiUrl,timeout: options.timeoutMs ?? 300000,@@ -52,6 +55,11 @@ export class HttpClient {return this.apiKey;}+ /** The attribution origin stamped into requests that do not set their own. */+ getOrigin(): string {+ return this.origin;+ }+/** The client-wide default request timeout, for callers that need to* extend a specific request beyond it (e.g. scrape auto-resume). */getTimeoutMs(): number {@@ -61,7 +69,6 @@ export class HttpClient {private async request<T = any>(config: AxiosRequestConfig,): Promise<AxiosResponse<T>> {- const version = getVersion();config.headers = {...(config.headers || {}),};@@ -89,9 +96,9 @@ export class HttpClient {cfg.data = {...data,origin:- typeof data.origin === "string" && data.origin.includes("mcp")+ typeof data.origin === "string" && data.origin? data.origin- : `js-sdk@${version}`,+ : this.origin,};}diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 9814f52e6..e18b8f052 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -39,6 +39,16 @@ crawl_status = firecrawl.crawl(print(crawl_status)```+### Request origin++Requests carry an `origin` tag that identifies the client. Pass the `origin` argument to set the client default. Per-call overrides are available on methods that explicitly accept an `origin` argument, such as `interact`.++```python+firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY", origin="[email protected]")+```++By default the tag is `python-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `python-sdk.<package>@<version>` instead, for example `python-sdk.some-agent@<version>`. Application code, REPLs and notebooks keep the plain default. Detection runs once when the client is created and sends only the installed distribution name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.+### Scraping a URLTo scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.pyindex 971b19e90..b746c655b 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py@@ -1,5 +1,8 @@"""Unit tests for configurable request origin attribution."""+import importlib+import sys+from firecrawl.v2.utils.http_client import HttpClientfrom firecrawl.v2.utils.http_client_async import AsyncHttpClientfrom firecrawl.v2.utils.get_version import get_version@@ -144,3 +147,69 @@ def test_research_get_query_carries_client_origin():client = FakeClient()research.search_papers(client, "transformers")assert "origin=arcade-mcp" in client.path+++def _install_fake_package(tmp_path, monkeypatch, install_dir="site-packages"):+ from firecrawl.v2.utils import caller_origin++ package = tmp_path / install_dir / "wrapper_pkg"+ package.mkdir(parents=True)+ (package / "__init__.py").write_text(+ "from firecrawl.v2.utils.http_client import HttpClient\n"+ "def make(**kwargs):\n"+ " return HttpClient(api_key='fc-test', api_url='https://api.firecrawl.dev', **kwargs)\n"+ )+ monkeypatch.syspath_prepend(str(tmp_path / install_dir))+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {"wrapper_pkg": ["Wrapper_Pkg.Extras"]})+ monkeypatch.delenv("FIRECRAWL_DISABLE_CALLER_ORIGIN", raising=False)+ monkeypatch.delitem(sys.modules, "wrapper_pkg", raising=False)+ return importlib.import_module("wrapper_pkg")+++def test_default_origin_names_installed_wrapper_package(tmp_path, monkeypatch):+ wrapper = _install_fake_package(tmp_path, monkeypatch)+ assert wrapper.make().origin == f"python-sdk.wrapper-pkg-extras@{get_version()}"+++def test_default_origin_reads_dist_packages(tmp_path, monkeypatch):+ wrapper = _install_fake_package(tmp_path, monkeypatch, install_dir="dist-packages")+ assert wrapper.make().origin == f"python-sdk.wrapper-pkg-extras@{get_version()}"+++def test_explicit_origin_wins_over_wrapper_default(tmp_path, monkeypatch):+ wrapper = _install_fake_package(tmp_path, monkeypatch)+ assert wrapper.make(origin="custom").origin == "custom"+++def test_wrapper_origin_can_be_disabled(tmp_path, monkeypatch):+ wrapper = _install_fake_package(tmp_path, monkeypatch)+ monkeypatch.setenv("FIRECRAWL_DISABLE_CALLER_ORIGIN", "1")+ assert wrapper.make().origin == f"python-sdk@{get_version()}"+++def test_unmapped_package_keeps_plain_origin(tmp_path, monkeypatch):+ from firecrawl.v2.utils import caller_origin++ wrapper = _install_fake_package(tmp_path, monkeypatch)+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {})+ assert wrapper.make().origin == f"python-sdk@{get_version()}"+++def test_detection_failure_keeps_plain_origin(tmp_path, monkeypatch):+ from firecrawl.v2.utils import caller_origin++ wrapper = _install_fake_package(tmp_path, monkeypatch)++ def boom():+ raise RuntimeError("metadata unavailable")++ monkeypatch.setattr(caller_origin, "_distributions", boom)+ assert wrapper.make().origin == f"python-sdk@{get_version()}"+++def test_playground_named_package_keeps_plain_origin(tmp_path, monkeypatch):+ from firecrawl.v2.utils import caller_origin++ wrapper = _install_fake_package(tmp_path, monkeypatch)+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {"wrapper_pkg": ["my-playground-kit"]})+ assert wrapper.make().origin == f"python-sdk@{get_version()}"diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 0d019df02..e8e8f35c9 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -262,13 +262,16 @@ class Firecrawl:max_retries: Maximum number of retries for failed requests (default: 3)backoff_factor: Exponential backoff factor for retries (default: 0.5)origin: Attribution string stamped into API request payloads- (defaults to ``python-sdk@<version>``)+ (defaults to ``python-sdk@<version>``, or+ ``python-sdk.<package>@<version>`` when an installed package+ creates the client; set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to+ keep the plain default)"""self.api_key = api_keyself.api_url = api_url# Initialize version-specific clients- self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None+ self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url, origin=origin) if V1FirecrawlApp else Noneself._v2_client = V2FirecrawlClient(api_key=api_key,api_url=api_url,@@ -413,7 +416,7 @@ class AsyncFirecrawl:self.api_url = api_url# Initialize version-specific clients- self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None+ self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url, origin=origin) if AsyncV1FirecrawlApp else Noneself._v2_client = AsyncFirecrawlClient(api_key=api_key,api_url=api_url,diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex e82ee6c07..c774b4620 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -25,6 +25,8 @@ import aiohttpimport asynciofrom ..v2.utils.api_origin import pin_to_api_origin+from ..v2.utils.caller_origin import default_origin+from ..v2.utils.get_version import get_version as get_sdk_versionlogger : logging.Logger = logging.getLogger("firecrawl")@@ -525,14 +527,16 @@ class V1FirecrawlApp:"Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.")- def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:+ def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None, origin: Optional[str] = None) -> None:"""Initialize the V1FirecrawlApp instance with API key, API URL.Args:api_key (Optional[str]): API key for authenticating with the Firecrawl API.api_url (Optional[str]): Base URL for the Firecrawl API.+ origin (Optional[str]): Attribution string stamped into API request payloads."""+ self.origin = origin or default_origin(get_sdk_version())self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')@@ -612,7 +616,7 @@ class V1FirecrawlApp:# Build scrape parametersscrape_params = {'url': url,- 'origin': f"python-sdk@{version}"+ 'origin': self.origin}# Add optional parameters if provided@@ -763,7 +767,7 @@ class V1FirecrawlApp:# Create final params objectfinal_params = V1SearchParams(query=query, **search_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif _integration:params_dict['integration'] = _integration@@ -985,7 +989,7 @@ class V1FirecrawlApp:final_params = V1CrawlParams(**crawl_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif _integration:params_dict['integration'] = _integration@@ -1113,7 +1117,7 @@ class V1FirecrawlApp:final_params = V1CrawlParams(**crawl_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.origin# Make requestheaders = self._prepare_headers(idempotency_key)@@ -1416,7 +1420,7 @@ class V1FirecrawlApp:final_params = V1MapParams(**map_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif _integration:params_dict['integration'] = _integration@@ -1563,7 +1567,7 @@ class V1FirecrawlApp:final_params = V1ScrapeParams(**scrape_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urls- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])@@ -1704,7 +1708,7 @@ class V1FirecrawlApp:final_params = V1ScrapeParams(**scrape_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urls- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])@@ -1840,7 +1844,7 @@ class V1FirecrawlApp:final_params = V1ScrapeParams(**scrape_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urls- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])@@ -2022,7 +2026,7 @@ class V1FirecrawlApp:'enableWebSearch': enable_web_search,'showSources': show_sources,'schema': schema,- 'origin': f'python-sdk@{get_version()}'+ 'origin': self.origin}# Only add prompt and systemPrompt if they exist@@ -2180,7 +2184,7 @@ class V1FirecrawlApp:'enableWebSearch': enable_web_search,'showSources': show_sources,'schema': schema,- 'origin': f'python-sdk@{version}'+ 'origin': self.origin}if prompt:@@ -2325,7 +2329,7 @@ class V1FirecrawlApp:headers = self._prepare_headers()json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}- json_data['origin'] = f"python-sdk@{version}"+ json_data['origin'] = self.origintry:req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)@@ -2790,7 +2794,7 @@ class V1FirecrawlApp:headers = self._prepare_headers()json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}- json_data['origin'] = f"python-sdk@{version}"+ json_data['origin'] = self.origin# Handle json options schema if presentif 'jsonOptions' in json_data:@@ -3382,9 +3386,9 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):Provides non-blocking alternatives to all V1FirecrawlApp operations."""- def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.dev"):+ def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.dev", origin: Optional[str] = None):# Reuse V1 helpers (_prepare_headers, _validate_kwargs, _ensure_schema_dict, _get_error_message)- super().__init__(api_key=api_key, api_url=api_url)+ super().__init__(api_key=api_key, api_url=api_url, origin=origin)async def _async_request(self,@@ -3690,7 +3694,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):# Build scrape parametersscrape_params = {'url': url,- 'origin': f"python-sdk@{version}"+ 'origin': self.origin}# Add optional parameters if provided and not None@@ -3870,7 +3874,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):final_params = V1ScrapeParams(**scrape_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urls- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])@@ -4012,7 +4016,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):final_params = V1ScrapeParams(**scrape_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['urls'] = urls- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originif 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])@@ -4136,7 +4140,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):final_params = V1CrawlParams(**crawl_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.origin# Make requestheaders = self._prepare_headers(idempotency_key)response = await self._async_post_request(@@ -4251,7 +4255,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):final_params = V1CrawlParams(**crawl_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.origin# Make requestheaders = self._prepare_headers(idempotency_key)@@ -4446,7 +4450,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):final_params = V1MapParams(**map_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)params_dict['url'] = url- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.origin# Make requestendpoint = f'/v1/map'@@ -4527,7 +4531,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):'enableWebSearch': enable_web_search,'showSources': show_sources,'schema': schema,- 'origin': f'python-sdk@{get_version()}'+ 'origin': self.origin}# Only add prompt and systemPrompt if they exist@@ -4793,14 +4797,14 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):if schema:schema = self._ensure_schema_dict(schema)- request_data = V1ExtractResponse(- urls=urls or [],- allowExternalLinks=allow_external_links,- enableWebSearch=enable_web_search,- showSources=show_sources,- schema=schema,- origin=f'python-sdk@{version}'- )+ request_data = {+ 'urls': urls or [],+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,+ 'schema': schema,+ 'origin': self.origin,+ }if prompt:request_data['prompt'] = prompt@@ -4941,7 +4945,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):headers = self._prepare_headers()json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}- json_data['origin'] = f"python-sdk@{version}"+ json_data['origin'] = self.origintry:return await self._async_post_request(@@ -5153,7 +5157,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):headers = self._prepare_headers()json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}- json_data['origin'] = f"python-sdk@{version}"+ json_data['origin'] = self.origintry:return await self._async_post_request(@@ -5279,7 +5283,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):# Create final params objectfinal_params = V1SearchParams(query=query, **search_params)params_dict = final_params.dict(by_alias=True, exclude_none=True)- params_dict['origin'] = f"python-sdk@{version}"+ params_dict['origin'] = self.originreturn await self._async_post_request(f"{self.api_url}/v1/search",diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 3bc02800a..e7e3b2a76 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -133,7 +133,10 @@ class FirecrawlClient:max_retries: Maximum number of retries for failed requestsbackoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)origin: Attribution string stamped into API request payloads- (defaults to ``python-sdk@<version>``)+ (defaults to ``python-sdk@<version>``, or+ ``python-sdk.<package>@<version>`` when an installed package+ creates the client; set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to+ keep the plain default)"""if api_key is None:api_key = os.getenv("FIRECRAWL_API_KEY")diff --git a/apps/python-sdk/firecrawl/v2/utils/caller_origin.py b/apps/python-sdk/firecrawl/v2/utils/caller_origin.pynew file mode 100644index 000000000..224f5a276--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/caller_origin.py@@ -0,0 +1,66 @@+import functools+import os+import re+import sys+from typing import Optional++_SDK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+_INSTALL_DIRS = ("site-packages", "dist-packages")++[email protected]_cache(maxsize=None)+def _distributions() -> dict:+ from importlib import metadata++ if hasattr(metadata, "packages_distributions"):+ return dict(metadata.packages_distributions())+ mapping: dict = {}+ for dist in metadata.distributions():+ names = (dist.read_text("top_level.txt") or "").split()+ if not names:+ first = {f.parts[0] for f in dist.files or [] if f.parts}+ names = [+ part.split(".")[0]+ for part in first+ if part not in ("..", "__pycache__") and not part.endswith((".dist-info", ".egg-info"))+ ]+ for name in names:+ mapping.setdefault(name, []).append(dist.metadata["Name"])+ return mapping+++def _is_sdk_file(filename: str) -> bool:+ if filename.startswith("<"):+ return False+ path = os.path.abspath(filename)+ return path.startswith(_SDK_ROOT + os.sep) and f"{os.sep}__tests__{os.sep}" not in path+++def _caller_distribution() -> Optional[str]:+ frame = sys._getframe()+ while frame is not None and _is_sdk_file(frame.f_code.co_filename):+ frame = frame.f_back+ if frame is None:+ return None+ parts = os.path.abspath(frame.f_code.co_filename).split(os.sep)+ for index in range(len(parts) - 2, -1, -1):+ if parts[index] in _INSTALL_DIRS:+ top_level = parts[index + 1].split(".")[0]+ names = _distributions().get(top_level)+ return names[0] if names else None+ return None+++def default_origin(version: str) -> str:+ """Return ``python-sdk@<version>``, or ``python-sdk.<package>@<version>`` when an installed package creates the client."""+ plain = f"python-sdk@{version}"+ try:+ if os.environ.get("FIRECRAWL_DISABLE_CALLER_ORIGIN") == "1":+ return plain+ name = _caller_distribution()+ slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:48].strip("-") if name else ""+ if not slug or "playground" in slug:+ return plain+ return f"python-sdk.{slug}@{version}"+ except Exception:+ return plaindiff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pyindex 20fd4d2a2..e8f9cc756 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -7,6 +7,7 @@ from typing import Dict, Any, Optionalfrom urllib.parse import urlparse, urljoinimport requestsfrom .api_origin import pin_to_api_origin+from .caller_origin import default_originfrom .get_version import get_versionversion = get_version()@@ -29,7 +30,7 @@ class HttpClient:self.max_retries = max_retriesself.backoff_factor = backoff_factor# Attribution string stamped into request payloads / research query params.- self.origin = origin or f"python-sdk@{version}"+ self.origin = origin or default_origin(version)def _build_url(self, endpoint: str) -> str:if urlparse(endpoint).netloc:diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex 4f08b13f9..4c8190876 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -2,6 +2,7 @@ import asyncioimport httpxfrom typing import Optional, Dict, Anyfrom .api_origin import pin_to_api_origin+from .caller_origin import default_originfrom .get_version import get_versionversion = get_version()@@ -23,7 +24,7 @@ class AsyncHttpClient:self.max_retries = max_retriesself.backoff_factor = backoff_factor# Attribution string stamped into request payloads / research query params.- self.origin = origin or f"python-sdk@{version}"+ self.origin = origin or default_origin(version)headers = {}