Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sdks): let wrapper libraries set and default the request origin (#5217)

Erik Engervall700f7d488dae
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 8bf7bb8d7..c45e89c50 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -33,6 +33,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
+import { isSdkOrigin } from "../../lib/sdk-origin";
async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
@@ -291,8 +292,7 @@ export async function extractController(
(await getTeamIdSyncB(req.auth.team_id)) &&
req.body.origin !== "api-sdk" &&
req.body.origin !== "website" &&
- !req.body.origin.startsWith("python-sdk@") &&
- !req.body.origin.startsWith("js-sdk@")
+ !isSdkOrigin(req.body.origin)
) {
return await oldExtract(req, res, extractId);
}
diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
index f0a6979ee..8465f3c49 100644
--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
@@ -230,6 +230,8 @@ describe("agentStatusController", () => {
["[email protected]", "spark-1-pro"],
// everything else sees the real model
["[email protected]", "spark-2"],
+ ["[email protected]", "spark-1-pro"],
+ ["[email protected]", "spark-2"],
["[email protected]", "spark-2"],
["[email protected]", "spark-2"],
["[email protected]", "spark-2"],
diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.ts
index b260ab77b..f91ff6551 100644
--- a/apps/api/src/controllers/v2/agent-status.ts
+++ b/apps/api/src/controllers/v2/agent-status.ts
@@ -22,7 +22,7 @@ import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";
// version regex is end-anchored so a prerelease of the fix (e.g.
// "4.37.1rc0", which may predate the Literal widening) also fails to
// parse and gets lied to rather than crashed.
-const PYTHON_SDK_ORIGIN = /^python-sdk@(.+)$/;
+const PYTHON_SDK_ORIGIN = /^python-sdk(?:\.[^@]*)?@(.+)$/;
const PYTHON_SDK_SPARK_2_FIX = [4, 37, 1];
function isIncompatiblePythonSdkOrigin(origin: unknown): boolean {
diff --git a/apps/api/src/lib/sdk-origin.test.ts b/apps/api/src/lib/sdk-origin.test.ts
new file mode 100644
index 000000000..59f3506f4
--- /dev/null
+++ b/apps/api/src/lib/sdk-origin.test.ts
@@ -0,0 +1,24 @@
+import { describe, expect, it } from "vitest";
+import { isSdkOrigin } from "./sdk-origin";
+
+describe("isSdkOrigin", () => {
+ it.each([
+ ])("accepts %s", origin => {
+ expect(isSdkOrigin(origin)).toBe(true);
+ });
+
+ it.each([
+ "api",
+ "website",
+ "mcp-server",
+ ])("rejects %s", origin => {
+ expect(isSdkOrigin(origin)).toBe(false);
+ });
+});
diff --git a/apps/api/src/lib/sdk-origin.ts b/apps/api/src/lib/sdk-origin.ts
new file mode 100644
index 000000000..dd3fac7fc
--- /dev/null
+++ b/apps/api/src/lib/sdk-origin.ts
@@ -0,0 +1,3 @@
+export function isSdkOrigin(origin: string): boolean {
+ return /^(python|js)-sdk[@.]/.test(origin);
+}
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md
index d97928399..c5c518215 100644
--- a/apps/js-sdk/firecrawl/README.md
+++ b/apps/js-sdk/firecrawl/README.md
@@ -37,6 +37,16 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
console.log(crawlResponse);
```
+### Request origin
+
+Requests carry an `origin` tag that identifies the client. Pass the `origin` option to set it. Set `origin` on a single call to override it for that call, except for `app.research` methods, which always use the client origin.
+
+```js
+const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY', origin: '[email protected]' });
+```
+
+By default the tag is `js-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `js-sdk.<package>@<version>` instead, for example `js-sdk.scope-name@<version>`. Application code, REPLs and bundled files without `node_modules` in their path keep the plain default. Detection runs once when the client is created and sends only the package name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.
+
### Scraping a URL
To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts
index 6dee3ed32..0690e569c 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent-hints.test.ts
@@ -14,6 +14,7 @@ const hints = [
];
function httpFor(body: object) {
return {
+ getOrigin: () => "js-sdk@test",
post: vi.fn<any>().mockResolvedValue({ status: 200, data: body }),
postMultipart: vi
.fn<any>()
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.ts
new file mode 100644
index 000000000..6d2901a89
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/origin.test.ts
@@ -0,0 +1,132 @@
+import { afterEach, beforeEach, describe, test, expect } from "vitest";
+import type { AxiosAdapter } from "axios";
+import { FirecrawlClient } from "../../../v2/client";
+import { Firecrawl } from "../../../index";
+import { getVersion } from "../../../v2/utils/getVersion";
+import { detectCallerPackage } from "../../../utils/callerOrigin";
+
+function withCapture<T extends { http: any }>(client: T) {
+ const bodies: Array<Record<string, unknown>> = [];
+ const adapter: AxiosAdapter = async config => {
+ bodies.push(JSON.parse(config.data));
+ return {
+ data: { success: true, data: { markdown: "# hello" } },
+ status: 200,
+ statusText: "OK",
+ headers: {},
+ config,
+ };
+ };
+ client.http.instance.defaults.adapter = adapter;
+ return { client, bodies };
+}
+
+describe("v2 request origin", () => {
+ const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
+ afterEach(() => {
+ if (previousOptOut === undefined) {
+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
+ } else {
+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
+ }
+ });
+
+ test("defaults to the plain SDK origin when application code creates the client", async () => {
+ const { client, bodies } = withCapture(new FirecrawlClient({ apiKey: "fc-test" }));
+ await client.scrape("https://example.com");
+ expect(bodies[0]!.origin).toBe(`js-sdk@${getVersion()}`);
+ });
+
+ test("the origin client option is stamped into request bodies", async () => {
+ const { client, bodies } = withCapture(
+ new FirecrawlClient({ apiKey: "fc-test", origin: "[email protected]" }),
+ );
+ await client.scrape("https://example.com");
+ expect(bodies[0]!.origin).toBe("[email protected]");
+ });
+
+ test("a per-call origin wins over the client origin", async () => {
+ const { client, bodies } = withCapture(
+ new FirecrawlClient({ apiKey: "fc-test", origin: "[email protected]" }),
+ );
+ await client.scrape("https://example.com", { origin: "per-call" } as any);
+ await client.scrape("https://example.com", { origin: "mcp-server" } as any);
+ expect(bodies.map(b => b.origin)).toEqual(["per-call", "mcp-server"]);
+ });
+
+ test("the unified client forwards the origin option to v1", () => {
+ const app = new Firecrawl({ apiKey: "fc-test", origin: "[email protected]" });
+ expect((app.v1 as any).clientOrigin).toBe("[email protected]");
+ });
+
+});
+
+describe("detectCallerPackage", () => {
+ const sdkFrames = [
+ "Error",
+ " at detectCallerPackage (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:10:5)",
+ " at new FirecrawlClient (file:///srv/app/node_modules/@mendable/firecrawl-js/dist/index.js:20:9)",
+ ];
+ const stackFrom = (...callers: string[]) =>
+ [...sdkFrames, ...callers.map(c => ` at ${c}`)].join("\n");
+ const previousOptOut = process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
+
+ beforeEach(() => {
+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
+ });
+
+ afterEach(() => {
+ if (previousOptOut === undefined) {
+ delete process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN;
+ } else {
+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = previousOptOut;
+ }
+ });
+
+ test("names the scoped package that created the client", () => {
+ const stack = stackFrom(
+ "make (file:///srv/app/node_modules/@Scope/Some_Agent/dist/index.js:3:12)",
+ "main (file:///srv/app/src/main.js:1:1)",
+ );
+ expect(detectCallerPackage(stack)).toBe("scope-some-agent");
+ });
+
+ test("uses the last node_modules segment of a nested install", () => {
+ const stack = stackFrom(
+ "make (/srv/app/node_modules/.pnpm/[email protected]/node_modules/inner-pkg/lib/index.cjs:3:12)",
+ );
+ expect(detectCallerPackage(stack)).toBe("inner-pkg");
+ });
+
+ test("handles windows paths with parentheses and anonymous frames", () => {
+ const stack = stackFrom(
+ "Array.map (<anonymous>)",
+ "C:\\Program Files (x86)\\app\\node_modules\\wrapper\\index.js:4:2",
+ );
+ expect(detectCallerPackage(stack)).toBe("wrapper");
+ });
+
+ test("returns null when application code created the client", () => {
+ const stack = stackFrom(
+ "main (file:///srv/app/src/main.js:1:1)",
+ "make (file:///srv/app/node_modules/wrapper/index.js:3:12)",
+ );
+ expect(detectCallerPackage(stack)).toBeNull();
+ });
+
+ test("returns null for a package whose slug would read as another traffic class", () => {
+ const stack = stackFrom("make (/srv/app/node_modules/my-playground-kit/index.js:3:12)");
+ expect(detectCallerPackage(stack)).toBeNull();
+ });
+
+ test("returns null for a bundle without node_modules in its path", () => {
+ expect(detectCallerPackage(stackFrom("make (/srv/app/dist/bundle.js:1:1)"))).toBeNull();
+ });
+
+ test("FIRECRAWL_DISABLE_CALLER_ORIGIN turns detection off", () => {
+ const stack = stackFrom("make (/srv/app/node_modules/wrapper/index.js:3:12)");
+ expect(detectCallerPackage(stack)).toBe("wrapper");
+ process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN = "1";
+ expect(detectCallerPackage(stack)).toBeNull();
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts
index 088e93d22..5dbc7cbe1 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts
@@ -12,6 +12,7 @@ function makeClient(
) {
const calls: string[] = [];
const http = {
+ getOrigin: () => "js-sdk@test",
get: async (url: string) => {
calls.push(url);
return responder(url);
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index a52be2453..0250f0826 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -37,6 +37,7 @@ export class Firecrawl extends V2 {
this._v1Opts = {
apiKey: resolved.apiKey,
apiUrl: resolved.apiUrl,
+ origin: this.http.getOrigin(),
};
}
diff --git a/apps/js-sdk/firecrawl/src/utils/callerOrigin.ts b/apps/js-sdk/firecrawl/src/utils/callerOrigin.ts
new file mode 100644
index 000000000..b32d35f8d
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/utils/callerOrigin.ts
@@ -0,0 +1,74 @@
+const NODE_MODULES = "/node_modules/";
+
+function packageOf(file: string): string | null {
+ let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
+ try {
+ path = decodeURIComponent(path);
+ } catch {}
+ const at = path.lastIndexOf(NODE_MODULES);
+ if (at < 0) return null;
+ const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
+ const name = first?.startsWith("@") ? (second ? `${first}/${second}` : "") : first;
+ return name && !name.startsWith(".") ? name : null;
+}
+
+/** File of a V8 stack line, or null for frames without one (native, anonymous, runtime internals). */
+function frameFile(line: string): string | null {
+ const text = line.trim().replace(/^at /, "");
+ const open = text.indexOf(" (");
+ const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
+ const match = /^(.*):\d+:\d+$/.exec(location);
+ return match && !match[1]!.startsWith("node:") ? match[1]! : null;
+}
+
+// Downstream surface classification matches "playground" anywhere in an origin.
+function slugify(name: string): string {
+ return name
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, "-")
+ .replace(/^-+|-+$/g, "")
+ .slice(0, 48)
+ .replace(/-+$/, "");
+}
+
+/**
+ * Slug of the installed package that created the client, or null when the
+ * caller is application code or detection is disabled. Never throws.
+ */
+export function detectCallerPackage(stack?: string): string | null {
+ try {
+ if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
+ if (stack === undefined) {
+ const limit = Error.stackTraceLimit;
+ Error.stackTraceLimit = 50;
+ try {
+ stack = new Error().stack ?? "";
+ } finally {
+ Error.stackTraceLimit = limit;
+ }
+ }
+ let ownFile: string | null = null;
+ let ownPackage: string | null = null;
+ for (const line of stack.split("\n")) {
+ if (!/^\s*at /.test(line)) continue;
+ const file = frameFile(line);
+ if (!file) continue;
+ if (ownFile === null) {
+ ownFile = file;
+ ownPackage = packageOf(file);
+ }
+ if (file === ownFile) continue;
+ const name = packageOf(file);
+ if (name && name === ownPackage) continue;
+ const slug = name ? slugify(name) : "";
+ return slug && !slug.includes("playground") ? slug : null;
+ }
+ } catch {}
+ return null;
+}
+
+/** Origin sent with requests: `js-sdk@<version>`, or `js-sdk.<package>@<version>` for wrapper libraries. */
+export function defaultOrigin(version: string): string {
+ const caller = detectCallerPackage();
+ return caller ? `js-sdk.${caller}@${version}` : `js-sdk@${version}`;
+}
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index caba47551..2661a9fb8 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -4,15 +4,18 @@ import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
import { TypedEventTarget } from "typescript-event-target";
import type { RequiresAction } from "../v2/types";
import { pinToApiOrigin } from "../utils/apiOrigin";
+import { detectCallerPackage } from "../utils/callerOrigin";
/**
* Configuration interface for FirecrawlApp.
* @param apiKey - Optional API key for authentication.
* @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ * @param origin - Optional attribution tag stamped into request bodies.
*/
export interface FirecrawlAppConfig {
apiKey?: string | null;
apiUrl?: string | null;
+ origin?: string;
}
/**
@@ -666,6 +669,8 @@ export default class FirecrawlApp {
public apiKey: string;
public apiUrl: string;
public version: string = "1.25.1";
+ private readonly clientOrigin?: string;
+ private readonly callerPackage: string | null;
private isCloudService(url: string): boolean {
return url.includes('api.firecrawl.dev');
@@ -698,7 +703,7 @@ export default class FirecrawlApp {
* Initializes a new instance of the FirecrawlApp class.
* @param config - Configuration options for the FirecrawlApp instance.
*/
- constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
+ constructor({ apiKey = null, apiUrl = null, origin }: FirecrawlAppConfig) {
const baseUrl = apiUrl || "https://api.firecrawl.dev";
if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
@@ -707,9 +712,19 @@ export default class FirecrawlApp {
this.apiKey = apiKey || '';
this.apiUrl = baseUrl;
+ this.clientOrigin = origin || undefined;
+ this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
this.init();
}
+ private resolveOrigin(params?: any): string {
+ if (typeof params?.origin === "string" && params.origin) return params.origin;
+ if (this.clientOrigin) return this.clientOrigin;
+ return this.callerPackage
+ ? `js-sdk.${this.callerPackage}@${this.version}`
+ : `js-sdk@${this.version}`;
+ }
+
/**
* Scrapes a URL using the Firecrawl API.
* @param url - The URL to scrape.
@@ -724,7 +739,7 @@ export default class FirecrawlApp {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
} as AxiosRequestHeaders;
- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
if (jsonData?.extract?.schema) {
jsonData = {
...jsonData,
@@ -791,7 +806,7 @@ export default class FirecrawlApp {
lang: params?.lang ?? "en",
country: params?.country ?? "us",
location: params?.location,
- origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}`,
+ origin: this.resolveOrigin(params),
timeout: params?.timeout ?? 60000,
scrapeOptions: params?.scrapeOptions ?? { formats: [] },
};
@@ -855,7 +870,7 @@ export default class FirecrawlApp {
idempotencyKey?: string
): Promise<CrawlStatusResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/crawl`,
@@ -884,7 +899,7 @@ export default class FirecrawlApp {
idempotencyKey?: string
): Promise<CrawlResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/crawl`,
@@ -1060,7 +1075,7 @@ export default class FirecrawlApp {
*/
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
const headers = this.prepareHeaders();
- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
try {
const response: AxiosResponse = await this.postRequest(
@@ -1099,7 +1114,7 @@ export default class FirecrawlApp {
maxConcurrency?: number,
): Promise<BatchScrapeStatusResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
if (jsonData?.extract?.schema) {
jsonData = {
...jsonData,
@@ -1148,7 +1163,7 @@ export default class FirecrawlApp {
ignoreInvalidURLs?: boolean,
): Promise<BatchScrapeResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/batch/scrape`,
@@ -1312,7 +1327,7 @@ export default class FirecrawlApp {
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/extract`,
- { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
+ { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
headers
);
@@ -1372,7 +1387,7 @@ export default class FirecrawlApp {
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/extract`,
- { ...jsonData, schema: jsonSchema, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` },
+ { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
headers
);
@@ -1740,7 +1755,7 @@ export default class FirecrawlApp {
*/
async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
const headers = this.prepareHeaders();
- let jsonData: any = { query, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { query, ...params, origin: this.resolveOrigin(params) };
if (jsonData?.jsonOptions?.schema) {
jsonData = {
@@ -1888,7 +1903,7 @@ export default class FirecrawlApp {
async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
const headers = this.prepareHeaders();
try {
- let jsonData: any = { topic, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { topic, ...params, origin: this.resolveOrigin(params) };
const response: AxiosResponse = await this.postRequest(
`${this.apiUrl}/v1/deep-research`,
jsonData,
@@ -2003,7 +2018,7 @@ export default class FirecrawlApp {
*/
async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
const headers = this.prepareHeaders();
- let jsonData: any = { url, ...params, origin: typeof (params as any)?.origin === "string" && (params as any).origin.includes("mcp") ? (params as any).origin : `js-sdk@${this.version}` };
+ let jsonData: any = { url, ...params, origin: this.resolveOrigin(params) };
try {
const response: AxiosResponse = await this.postRequest(
`${this.apiUrl}/v1/llmstxt`,
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 41d3f1da9..b9be6b349 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -101,6 +101,8 @@ import type {
GetMonitorCheckOptions,
UpdateMonitorRequest,
} from "./types";
+import { defaultOrigin } from "../utils/callerOrigin";
+import { getVersion } from "./utils/getVersion";
import { Watcher } from "./watcher";
import type { WatcherOptions } from "./watcher";
import * as zt from "zod";
@@ -130,6 +132,12 @@ export interface FirecrawlClientOptions {
maxRetries?: number;
/** Exponential backoff factor for retries (optional). */
backoffFactor?: number;
+ /**
+ * Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
+ * or `js-sdk.<package>@<version>` when an installed package creates the client
+ * (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
+ */
+ origin?: string;
}
/** Accepts a plain API key string or a full options object. */
@@ -140,7 +148,7 @@ export type FirecrawlClientInput = FirecrawlClientOptions | string;
*/
export class FirecrawlClient {
- private readonly http: HttpClient;
+ protected readonly http: HttpClient;
private _research?: ResearchClient;
private isCloudService(url: string): boolean {
@@ -168,6 +176,7 @@ export class FirecrawlClient {
timeoutMs: opts.timeoutMs,
maxRetries: opts.maxRetries,
backoffFactor: opts.backoffFactor,
+ origin: opts.origin || defaultOrigin(getVersion()),
});
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
index 183ca6730..5436c3e32 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
@@ -6,7 +6,6 @@ import {
throwForBadResponse,
normalizeAxiosError,
} from "../utils/errorHandler";
-import { getVersion } from "../utils/getVersion";
function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
if (typeof Blob !== "undefined" && input instanceof Blob) {
@@ -57,13 +56,9 @@ export async function parse(
if (options) ensureValidParseOptions(options);
- const version = getVersion();
const normalizedOptions: ParseOptions = {
...(options ?? {}),
- origin:
- typeof options?.origin === "string" && options.origin.includes("mcp")
- ? options.origin
- : (options?.origin ?? `js-sdk@${version}`),
+ origin: options?.origin || http.getOrigin(),
};
const formData = new FormData();
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.ts
index 5292b2997..270336bb2 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts
@@ -12,7 +12,6 @@ import type {
import { SdkError } from "../types";
import { HttpClient } from "../utils/httpClient";
import { throwForBadResponse } from "../utils/errorHandler";
-import { getVersion } from "../utils/getVersion";
const BASE = "/v2/search/research";
@@ -32,10 +31,14 @@ function appendParam(
}
}
-function withQuery(path: string, params: URLSearchParams): string {
+function withQuery(
+ origin: string,
+ path: string,
+ params: URLSearchParams,
+): string {
// Research endpoints are GETs, so the POST-body origin injection in
// HttpClient never applies — attach it as a query param instead.
- params.append("origin", `js-sdk@${getVersion()}`);
+ params.append("origin", origin);
const qs = params.toString();
return qs ? `${path}?${qs}` : path;
}
@@ -121,7 +124,7 @@ export class ResearchClient {
appendParam(params, "to", options.to);
try {
const res = await this.http.get<SearchPapersResponse>(
- withQuery(`${BASE}/papers`, params),
+ withQuery(this.http.getOrigin(), `${BASE}/papers`, params),
);
if (res.status !== 200) throwForBadResponse(res, "search papers");
return res.data;
@@ -161,7 +164,13 @@ export class ResearchClient {
try {
const res = await this.http.get<
PaperMetadataResponse | ReadPaperResponse
- >(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
+ >(
+ withQuery(
+ this.http.getOrigin(),
+ `${BASE}/papers/${encodeURIComponent(id)}`,
+ params,
+ ),
+ );
if (res.status !== 200) throwForBadResponse(res, "get paper");
return res.data;
} catch (err) {
@@ -191,7 +200,11 @@ export class ResearchClient {
appendParam(params, "anchor", options.anchor);
try {
const res = await this.http.get<SimilarPapersResponse>(
- withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params),
+ withQuery(
+ this.http.getOrigin(),
+ `${BASE}/papers/${encodeURIComponent(id)}/similar`,
+ params,
+ ),
);
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
return res.data;
@@ -226,7 +239,7 @@ export class ResearchClient {
appendParam(params, "k", options.k);
try {
const res = await this.http.get<GitHubSearchResponse>(
- withQuery(`${BASE}/github`, params),
+ withQuery(this.http.getOrigin(), `${BASE}/github`, params),
);
if (res.status !== 200) throwForBadResponse(res, "search github");
return res.data;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index b59c047a8..495565e85 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -12,6 +12,7 @@ export interface HttpClientOptions {
timeoutMs?: number;
maxRetries?: number;
backoffFactor?: number; // seconds factor for 0.5, 1, 2...
+ origin?: string;
}
export interface RequestOptions {
@@ -25,12 +26,14 @@ export class HttpClient {
private readonly apiUrl: string;
private readonly maxRetries: number;
private readonly backoffFactor: number;
+ private readonly origin: string;
constructor(options: HttpClientOptions) {
this.apiKey = options.apiKey;
this.apiUrl = options.apiUrl.replace(/\/$/, "");
this.maxRetries = options.maxRetries ?? 3;
this.backoffFactor = options.backoffFactor ?? 0.5;
+ this.origin = options.origin || `js-sdk@${getVersion()}`;
this.instance = axios.create({
baseURL: this.apiUrl,
timeout: options.timeoutMs ?? 300000,
@@ -52,6 +55,11 @@ export class HttpClient {
return this.apiKey;
}
+ /** The attribution origin stamped into requests that do not set their own. */
+ getOrigin(): string {
+ return this.origin;
+ }
+
/** The client-wide default request timeout, for callers that need to
* extend a specific request beyond it (e.g. scrape auto-resume). */
getTimeoutMs(): number {
@@ -61,7 +69,6 @@ export class HttpClient {
private async request<T = any>(
config: AxiosRequestConfig,
): Promise<AxiosResponse<T>> {
- const version = getVersion();
config.headers = {
...(config.headers || {}),
};
@@ -89,9 +96,9 @@ export class HttpClient {
cfg.data = {
...data,
origin:
- typeof data.origin === "string" && data.origin.includes("mcp")
+ typeof data.origin === "string" && data.origin
? data.origin
- : `js-sdk@${version}`,
+ : this.origin,
};
}
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md
index 9814f52e6..e18b8f052 100644
--- a/apps/python-sdk/README.md
+++ b/apps/python-sdk/README.md
@@ -39,6 +39,16 @@ crawl_status = firecrawl.crawl(
print(crawl_status)
```
+### Request origin
+
+Requests carry an `origin` tag that identifies the client. Pass the `origin` argument to set the client default. Per-call overrides are available on methods that explicitly accept an `origin` argument, such as `interact`.
+
+```python
+firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY", origin="[email protected]")
+```
+
+By default the tag is `python-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `python-sdk.<package>@<version>` instead, for example `python-sdk.some-agent@<version>`. Application code, REPLs and notebooks keep the plain default. Detection runs once when the client is created and sends only the installed distribution name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.
+
### Scraping a URL
To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py
index 971b19e90..b746c655b 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py
@@ -1,5 +1,8 @@
"""Unit tests for configurable request origin attribution."""
+import importlib
+import sys
+
from firecrawl.v2.utils.http_client import HttpClient
from firecrawl.v2.utils.http_client_async import AsyncHttpClient
from firecrawl.v2.utils.get_version import get_version
@@ -144,3 +147,69 @@ def test_research_get_query_carries_client_origin():
client = FakeClient()
research.search_papers(client, "transformers")
assert "origin=arcade-mcp" in client.path
+
+
+def _install_fake_package(tmp_path, monkeypatch, install_dir="site-packages"):
+ from firecrawl.v2.utils import caller_origin
+
+ package = tmp_path / install_dir / "wrapper_pkg"
+ package.mkdir(parents=True)
+ (package / "__init__.py").write_text(
+ "from firecrawl.v2.utils.http_client import HttpClient\n"
+ "def make(**kwargs):\n"
+ " return HttpClient(api_key='fc-test', api_url='https://api.firecrawl.dev', **kwargs)\n"
+ )
+ monkeypatch.syspath_prepend(str(tmp_path / install_dir))
+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {"wrapper_pkg": ["Wrapper_Pkg.Extras"]})
+ monkeypatch.delenv("FIRECRAWL_DISABLE_CALLER_ORIGIN", raising=False)
+ monkeypatch.delitem(sys.modules, "wrapper_pkg", raising=False)
+ return importlib.import_module("wrapper_pkg")
+
+
+def test_default_origin_names_installed_wrapper_package(tmp_path, monkeypatch):
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+ assert wrapper.make().origin == f"python-sdk.wrapper-pkg-extras@{get_version()}"
+
+
+def test_default_origin_reads_dist_packages(tmp_path, monkeypatch):
+ wrapper = _install_fake_package(tmp_path, monkeypatch, install_dir="dist-packages")
+ assert wrapper.make().origin == f"python-sdk.wrapper-pkg-extras@{get_version()}"
+
+
+def test_explicit_origin_wins_over_wrapper_default(tmp_path, monkeypatch):
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+ assert wrapper.make(origin="custom").origin == "custom"
+
+
+def test_wrapper_origin_can_be_disabled(tmp_path, monkeypatch):
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+ monkeypatch.setenv("FIRECRAWL_DISABLE_CALLER_ORIGIN", "1")
+ assert wrapper.make().origin == f"python-sdk@{get_version()}"
+
+
+def test_unmapped_package_keeps_plain_origin(tmp_path, monkeypatch):
+ from firecrawl.v2.utils import caller_origin
+
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {})
+ assert wrapper.make().origin == f"python-sdk@{get_version()}"
+
+
+def test_detection_failure_keeps_plain_origin(tmp_path, monkeypatch):
+ from firecrawl.v2.utils import caller_origin
+
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+
+ def boom():
+ raise RuntimeError("metadata unavailable")
+
+ monkeypatch.setattr(caller_origin, "_distributions", boom)
+ assert wrapper.make().origin == f"python-sdk@{get_version()}"
+
+
+def test_playground_named_package_keeps_plain_origin(tmp_path, monkeypatch):
+ from firecrawl.v2.utils import caller_origin
+
+ wrapper = _install_fake_package(tmp_path, monkeypatch)
+ monkeypatch.setattr(caller_origin, "_distributions", lambda: {"wrapper_pkg": ["my-playground-kit"]})
+ assert wrapper.make().origin == f"python-sdk@{get_version()}"
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index 0d019df02..e8e8f35c9 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -262,13 +262,16 @@ class Firecrawl:
max_retries: Maximum number of retries for failed requests (default: 3)
backoff_factor: Exponential backoff factor for retries (default: 0.5)
origin: Attribution string stamped into API request payloads
- (defaults to ``python-sdk@<version>``)
+ (defaults to ``python-sdk@<version>``, or
+ ``python-sdk.<package>@<version>`` when an installed package
+ creates the client; set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to
+ keep the plain default)
"""
self.api_key = api_key
self.api_url = api_url
# Initialize version-specific clients
- self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None
+ self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url, origin=origin) if V1FirecrawlApp else None
self._v2_client = V2FirecrawlClient(
api_key=api_key,
api_url=api_url,
@@ -413,7 +416,7 @@ class AsyncFirecrawl:
self.api_url = api_url
# Initialize version-specific clients
- self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None
+ self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url, origin=origin) if AsyncV1FirecrawlApp else None
self._v2_client = AsyncFirecrawlClient(
api_key=api_key,
api_url=api_url,
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index e82ee6c07..c774b4620 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -25,6 +25,8 @@ import aiohttp
import asyncio
from ..v2.utils.api_origin import pin_to_api_origin
+from ..v2.utils.caller_origin import default_origin
+from ..v2.utils.get_version import get_version as get_sdk_version
logger : logging.Logger = logging.getLogger("firecrawl")
@@ -525,14 +527,16 @@ class V1FirecrawlApp:
"Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported."
)
- def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None) -> None:
+ def __init__(self, api_key: Optional[str] = None, api_url: Optional[str] = None, origin: Optional[str] = None) -> None:
"""
Initialize the V1FirecrawlApp instance with API key, API URL.
Args:
api_key (Optional[str]): API key for authenticating with the Firecrawl API.
api_url (Optional[str]): Base URL for the Firecrawl API.
+ origin (Optional[str]): Attribution string stamped into API request payloads.
"""
+ self.origin = origin or default_origin(get_sdk_version())
self.api_key = api_key or os.getenv('FIRECRAWL_API_KEY')
self.api_url = api_url or os.getenv('FIRECRAWL_API_URL', 'https://api.firecrawl.dev')
@@ -612,7 +616,7 @@ class V1FirecrawlApp:
# Build scrape parameters
scrape_params = {
'url': url,
- 'origin': f"python-sdk@{version}"
+ 'origin': self.origin
}
# Add optional parameters if provided
@@ -763,7 +767,7 @@ class V1FirecrawlApp:
# Create final params object
final_params = V1SearchParams(query=query, **search_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if _integration:
params_dict['integration'] = _integration
@@ -985,7 +989,7 @@ class V1FirecrawlApp:
final_params = V1CrawlParams(**crawl_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if _integration:
params_dict['integration'] = _integration
@@ -1113,7 +1117,7 @@ class V1FirecrawlApp:
final_params = V1CrawlParams(**crawl_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
# Make request
headers = self._prepare_headers(idempotency_key)
@@ -1416,7 +1420,7 @@ class V1FirecrawlApp:
final_params = V1MapParams(**map_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if _integration:
params_dict['integration'] = _integration
@@ -1563,7 +1567,7 @@ class V1FirecrawlApp:
final_params = V1ScrapeParams(**scrape_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
@@ -1704,7 +1708,7 @@ class V1FirecrawlApp:
final_params = V1ScrapeParams(**scrape_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
@@ -1840,7 +1844,7 @@ class V1FirecrawlApp:
final_params = V1ScrapeParams(**scrape_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
@@ -2022,7 +2026,7 @@ class V1FirecrawlApp:
'enableWebSearch': enable_web_search,
'showSources': show_sources,
'schema': schema,
- 'origin': f'python-sdk@{get_version()}'
+ 'origin': self.origin
}
# Only add prompt and systemPrompt if they exist
@@ -2180,7 +2184,7 @@ class V1FirecrawlApp:
'enableWebSearch': enable_web_search,
'showSources': show_sources,
'schema': schema,
- 'origin': f'python-sdk@{version}'
+ 'origin': self.origin
}
if prompt:
@@ -2325,7 +2329,7 @@ class V1FirecrawlApp:
headers = self._prepare_headers()
json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
- json_data['origin'] = f"python-sdk@{version}"
+ json_data['origin'] = self.origin
try:
req = self._post_request(f'{self.api_url}/v1/llmstxt', json_data, headers)
@@ -2790,7 +2794,7 @@ class V1FirecrawlApp:
headers = self._prepare_headers()
json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
- json_data['origin'] = f"python-sdk@{version}"
+ json_data['origin'] = self.origin
# Handle json options schema if present
if 'jsonOptions' in json_data:
@@ -3382,9 +3386,9 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
Provides non-blocking alternatives to all V1FirecrawlApp operations.
"""
- def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.dev"):
+ def __init__(self, api_key: str, api_url: str = "https://api.firecrawl.dev", origin: Optional[str] = None):
# Reuse V1 helpers (_prepare_headers, _validate_kwargs, _ensure_schema_dict, _get_error_message)
- super().__init__(api_key=api_key, api_url=api_url)
+ super().__init__(api_key=api_key, api_url=api_url, origin=origin)
async def _async_request(
self,
@@ -3690,7 +3694,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
# Build scrape parameters
scrape_params = {
'url': url,
- 'origin': f"python-sdk@{version}"
+ 'origin': self.origin
}
# Add optional parameters if provided and not None
@@ -3870,7 +3874,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
final_params = V1ScrapeParams(**scrape_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
@@ -4012,7 +4016,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
final_params = V1ScrapeParams(**scrape_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['urls'] = urls
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
if 'extract' in params_dict and params_dict['extract'] and 'schema' in params_dict['extract']:
params_dict['extract']['schema'] = self._ensure_schema_dict(params_dict['extract']['schema'])
@@ -4136,7 +4140,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
final_params = V1CrawlParams(**crawl_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
# Make request
headers = self._prepare_headers(idempotency_key)
response = await self._async_post_request(
@@ -4251,7 +4255,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
final_params = V1CrawlParams(**crawl_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
# Make request
headers = self._prepare_headers(idempotency_key)
@@ -4446,7 +4450,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
final_params = V1MapParams(**map_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
params_dict['url'] = url
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
# Make request
endpoint = f'/v1/map'
@@ -4527,7 +4531,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
'enableWebSearch': enable_web_search,
'showSources': show_sources,
'schema': schema,
- 'origin': f'python-sdk@{get_version()}'
+ 'origin': self.origin
}
# Only add prompt and systemPrompt if they exist
@@ -4793,14 +4797,14 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
if schema:
schema = self._ensure_schema_dict(schema)
- request_data = V1ExtractResponse(
- urls=urls or [],
- allowExternalLinks=allow_external_links,
- enableWebSearch=enable_web_search,
- showSources=show_sources,
- schema=schema,
- origin=f'python-sdk@{version}'
- )
+ request_data = {
+ 'urls': urls or [],
+ 'allowExternalLinks': allow_external_links,
+ 'enableWebSearch': enable_web_search,
+ 'showSources': show_sources,
+ 'schema': schema,
+ 'origin': self.origin,
+ }
if prompt:
request_data['prompt'] = prompt
@@ -4941,7 +4945,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
headers = self._prepare_headers()
json_data = {'url': url, **params.dict(by_alias=True, exclude_none=True)}
- json_data['origin'] = f"python-sdk@{version}"
+ json_data['origin'] = self.origin
try:
return await self._async_post_request(
@@ -5153,7 +5157,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
headers = self._prepare_headers()
json_data = {'query': query, **research_params.dict(by_alias=True, exclude_none=True)}
- json_data['origin'] = f"python-sdk@{version}"
+ json_data['origin'] = self.origin
try:
return await self._async_post_request(
@@ -5279,7 +5283,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
# Create final params object
final_params = V1SearchParams(query=query, **search_params)
params_dict = final_params.dict(by_alias=True, exclude_none=True)
- params_dict['origin'] = f"python-sdk@{version}"
+ params_dict['origin'] = self.origin
return await self._async_post_request(
f"{self.api_url}/v1/search",
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 3bc02800a..e7e3b2a76 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -133,7 +133,10 @@ class FirecrawlClient:
max_retries: Maximum number of retries for failed requests
backoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)
origin: Attribution string stamped into API request payloads
- (defaults to ``python-sdk@<version>``)
+ (defaults to ``python-sdk@<version>``, or
+ ``python-sdk.<package>@<version>`` when an installed package
+ creates the client; set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to
+ keep the plain default)
"""
if api_key is None:
api_key = os.getenv("FIRECRAWL_API_KEY")
diff --git a/apps/python-sdk/firecrawl/v2/utils/caller_origin.py b/apps/python-sdk/firecrawl/v2/utils/caller_origin.py
new file mode 100644
index 000000000..224f5a276
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/caller_origin.py
@@ -0,0 +1,66 @@
+import functools
+import os
+import re
+import sys
+from typing import Optional
+
+_SDK_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+_INSTALL_DIRS = ("site-packages", "dist-packages")
+
+
[email protected]_cache(maxsize=None)
+def _distributions() -> dict:
+ from importlib import metadata
+
+ if hasattr(metadata, "packages_distributions"):
+ return dict(metadata.packages_distributions())
+ mapping: dict = {}
+ for dist in metadata.distributions():
+ names = (dist.read_text("top_level.txt") or "").split()
+ if not names:
+ first = {f.parts[0] for f in dist.files or [] if f.parts}
+ names = [
+ part.split(".")[0]
+ for part in first
+ if part not in ("..", "__pycache__") and not part.endswith((".dist-info", ".egg-info"))
+ ]
+ for name in names:
+ mapping.setdefault(name, []).append(dist.metadata["Name"])
+ return mapping
+
+
+def _is_sdk_file(filename: str) -> bool:
+ if filename.startswith("<"):
+ return False
+ path = os.path.abspath(filename)
+ return path.startswith(_SDK_ROOT + os.sep) and f"{os.sep}__tests__{os.sep}" not in path
+
+
+def _caller_distribution() -> Optional[str]:
+ frame = sys._getframe()
+ while frame is not None and _is_sdk_file(frame.f_code.co_filename):
+ frame = frame.f_back
+ if frame is None:
+ return None
+ parts = os.path.abspath(frame.f_code.co_filename).split(os.sep)
+ for index in range(len(parts) - 2, -1, -1):
+ if parts[index] in _INSTALL_DIRS:
+ top_level = parts[index + 1].split(".")[0]
+ names = _distributions().get(top_level)
+ return names[0] if names else None
+ return None
+
+
+def default_origin(version: str) -> str:
+ """Return ``python-sdk@<version>``, or ``python-sdk.<package>@<version>`` when an installed package creates the client."""
+ plain = f"python-sdk@{version}"
+ try:
+ if os.environ.get("FIRECRAWL_DISABLE_CALLER_ORIGIN") == "1":
+ return plain
+ name = _caller_distribution()
+ slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:48].strip("-") if name else ""
+ if not slug or "playground" in slug:
+ return plain
+ return f"python-sdk.{slug}@{version}"
+ except Exception:
+ return plain
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.py
index 20fd4d2a2..e8f9cc756 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py
@@ -7,6 +7,7 @@ from typing import Dict, Any, Optional
from urllib.parse import urlparse, urljoin
import requests
from .api_origin import pin_to_api_origin
+from .caller_origin import default_origin
from .get_version import get_version
version = get_version()
@@ -29,7 +30,7 @@ class HttpClient:
self.max_retries = max_retries
self.backoff_factor = backoff_factor
# Attribution string stamped into request payloads / research query params.
- self.origin = origin or f"python-sdk@{version}"
+ self.origin = origin or default_origin(version)
def _build_url(self, endpoint: str) -> str:
if urlparse(endpoint).netloc:
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
index 4f08b13f9..4c8190876 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
@@ -2,6 +2,7 @@ import asyncio
import httpx
from typing import Optional, Dict, Any
from .api_origin import pin_to_api_origin
+from .caller_origin import default_origin
from .get_version import get_version
version = get_version()
@@ -23,7 +24,7 @@ class AsyncHttpClient:
self.max_retries = max_retries
self.backoff_factor = backoff_factor
# Attribution string stamped into request payloads / research query params.
- self.origin = origin or f"python-sdk@{version}"
+ self.origin = origin or default_origin(version)
headers = {}