Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): includeDomains/excludeDomains (#3466)
Gergő Móricz105fc997bc30
diff --git a/apps/api/src/__tests__/lib/search-query-builder.test.ts b/apps/api/src/__tests__/lib/search-query-builder.test.tsindex 8da3d02f0..f746dd2c2 100644--- a/apps/api/src/__tests__/lib/search-query-builder.test.ts+++ b/apps/api/src/__tests__/lib/search-query-builder.test.ts@@ -110,6 +110,45 @@ describe("Search Query Builder", () => {expect(result.query).toBe("test");expect(result.categoryMap.size).toBe(0);});++ it("should add include domain filters", () => {+ const result = buildSearchQuery("web scraping", undefined, {+ includeDomains: ["firecrawl.dev", "docs.firecrawl.dev"],+ });+ expect(result.query).toBe(+ "web scraping (site:firecrawl.dev OR site:docs.firecrawl.dev)",+ );+ expect(result.categoryMap.size).toBe(0);+ });++ it("should add exclude domain filters", () => {+ const result = buildSearchQuery("web scraping", undefined, {+ excludeDomains: ["example.com", "spam.example.com"],+ });+ expect(result.query).toBe(+ "web scraping -site:example.com -site:spam.example.com",+ );+ expect(result.categoryMap.size).toBe(0);+ });++ it("should ignore empty domain filter arrays", () => {+ const result = buildSearchQuery("web scraping", undefined, {+ includeDomains: [],+ excludeDomains: [],+ });+ expect(result.query).toBe("web scraping");+ expect(result.categoryMap.size).toBe(0);+ });++ it("should combine categories with domain filters", () => {+ const result = buildSearchQuery("web scraping", ["github"], {+ includeDomains: ["firecrawl.dev"],+ });+ expect(result.query).toBe(+ "web scraping (site:github.com) (site:firecrawl.dev)",+ );+ expect(result.categoryMap.get("github.com")).toBe("github");+ });});describe("getCategoryFromUrl", () => {diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 78c54f3b2..840a2c7c7 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -478,7 +478,7 @@ export function expectMapToSucceed(response: Awaited<ReturnType<typeof map>>) {// Search API// =========================================-async function searchRaw(body: SearchRequestInput, identity: Identity) {+export async function searchRaw(body: SearchRequestInput, identity: Identity) {return await request(TEST_API_URL).post("/v2/search").set("Authorization", `Bearer ${identity.apiKey}`)@@ -494,6 +494,12 @@ function expectSearchToSucceed(expect(typeof response.body.data).toBe("object");}+function expectSearchToFail(response: Awaited<ReturnType<typeof searchRaw>>) {+ expect(response.statusCode).not.toBe(200);+ expect(response.body.success).toBe(false);+ expect(typeof response.body.error).toBe("string");+}+export async function search(body: SearchRequestInput,identity: Identity,@@ -503,6 +509,19 @@ export async function search(return raw.body.data;}+export async function searchWithFailure(+ body: SearchRequestInput,+ identity: Identity,+): Promise<{+ success: false;+ error: string;+ details?: unknown;+}> {+ const raw = await searchRaw(body, identity);+ expectSearchToFail(raw);+ return raw.body;+}+// =========================================// Billing API// =========================================diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex b91ab1551..60fdc5658 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -5,7 +5,7 @@ import {HAS_SEARCH,TEST_PRODUCTION,} from "../lib";-import { search, idmux, Identity } from "./lib";+import { search, searchWithFailure, idmux, Identity } from "./lib";import { config } from "../../../config";let identity: Identity;@@ -35,6 +35,48 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {60000,);+ it.concurrent(+ "works with includeDomains",+ async () => {+ const res = await search(+ {+ query: "firecrawl",+ includeDomains: ["firecrawl.dev"],+ limit: 3,+ },+ identity,+ );++ expect(res.web).toBeDefined();+ expect(res.web?.length).toBeGreaterThan(0);+ for (const result of res.web ?? []) {+ expect(result.url).toBeDefined();+ const hostname = new URL(result.url!).hostname;+ expect(+ hostname === "firecrawl.dev" || hostname.endsWith(".firecrawl.dev"),+ ).toBe(true);+ }+ },+ 60000,+ );++ it.concurrent(+ "rejects includeDomains with excludeDomains",+ async () => {+ const res = await searchWithFailure(+ {+ query: "firecrawl",+ includeDomains: ["firecrawl.dev"],+ excludeDomains: ["example.com"],+ },+ identity,+ );++ expect(res.error).toBe("Invalid request body");+ },+ 60000,+ );+it.concurrent("works with scrape",async () => {diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 98c91032d..4e97a5731 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -914,6 +914,44 @@ describe("V2 Types Validation", () => {expect(Array.isArray(result.categories)).toBe(true);});+ it("should accept search request with includeDomains", () => {+ const input: SearchRequestInput = {+ query: "test",+ includeDomains: [" Example.com ", "docs.example.com"],+ };++ const result = searchRequestSchema.parse(input);+ expect(result.includeDomains).toEqual([+ "example.com",+ "docs.example.com",+ ]);+ });++ it("should accept search request with excludeDomains", () => {+ const input: SearchRequestInput = {+ query: "test",+ excludeDomains: ["example.com", "spam.example.com"],+ };++ const result = searchRequestSchema.parse(input);+ expect(result.excludeDomains).toEqual([+ "example.com",+ "spam.example.com",+ ]);+ });++ it("should reject search request with includeDomains and excludeDomains", () => {+ const input: SearchRequestInput = {+ query: "test",+ includeDomains: ["example.com"],+ excludeDomains: ["spam.example.com"],+ };++ expect(() => searchRequestSchema.parse(input)).toThrow(+ "includeDomains and excludeDomains cannot both be specified",+ );+ });+it("should reject limit exceeding 100", () => {const input: SearchRequestInput = {query: "test",diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 74b87b2c8..74160e63e 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -127,6 +127,8 @@ export async function searchController(location: req.body.location,sources: req.body.sources as Array<{ type: string }>,categories: req.body.categories as CategoryOption[],+ includeDomains: req.body.includeDomains,+ excludeDomains: req.body.excludeDomains,enterprise: req.body.enterprise,scrapeOptions: req.body.scrapeOptions,timeout: req.body.timeout,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 961c123f7..f11bb094c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1703,6 +1703,18 @@ const pdfCategoryOptions = z.strictObject({type: z.literal("pdf"),});+const searchDomainSchema = z+ .string()+ .trim()+ .toLowerCase()+ .refine(+ value =>+ /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/.test(+ value,+ ),+ "Domain must be a valid hostname without protocol or path",+ );+export const searchRequestSchema = z.strictObject({query: z.string(),@@ -1738,6 +1750,8 @@ export const searchRequestSchema = z),]).optional(),+ includeDomains: z.array(searchDomainSchema).optional(),+ excludeDomains: z.array(searchDomainSchema).optional(),lang: z.string().optional().prefault("en"),enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional(),country: z.string().optional(),@@ -1790,6 +1804,10 @@ export const searchRequestSchema = z}).optional(),})+ .refine(+ x => !(x.includeDomains?.length && x.excludeDomains?.length),+ "includeDomains and excludeDomains cannot both be specified",+ ).refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {const country =diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex bcaaf2eb6..098ac2ae8 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -284,6 +284,10 @@ export async function x402SearchController(const { query: searchQuery, categoryMap } = buildSearchQuery(req.body.query,req.body.categories as CategoryOption[],+ {+ includeDomains: req.body.includeDomains,+ excludeDomains: req.body.excludeDomains,+ },);const searchResponse = (await search({diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex 9619d1ec8..8f1b2797b 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -10,6 +10,11 @@ interface CategoryInput {export type CategoryOption = string | CategoryInput;+interface DomainFilterOptions {+ includeDomains?: string[];+ excludeDomains?: string[];+}+interface QueryBuilderResult {query: string;categoryMap: Map<string, string>;@@ -42,20 +47,14 @@ const DEFAULT_RESEARCH_SITES = [export function buildSearchQuery(baseQuery: string,categories?: CategoryOption[],+ domainOptions: DomainFilterOptions = {},): QueryBuilderResult {const categoryMap = new Map<string, string>();- if (!categories || categories.length === 0) {- return {- query: baseQuery,- categoryMap,- };- }-const siteFilters: string[] = [];let hasPdfFilter = false;- for (const category of categories) {+ for (const category of categories ?? []) {if (typeof category === "string") {// Simple string formatif (category === "github") {@@ -101,8 +100,19 @@ export function buildSearchQuery(categoryFilter += " filetype:pdf";}+ const includeDomains = domainOptions.includeDomains ?? [];+ const excludeDomains = domainOptions.excludeDomains ?? [];+ const includeFilter =+ includeDomains.length > 0+ ? " (" + includeDomains.map(domain => `site:${domain}`).join(" OR ") + ")"+ : "";+ const excludeFilter =+ excludeDomains.length > 0+ ? " " + excludeDomains.map(domain => `-site:${domain}`).join(" ")+ : "";+return {- query: baseQuery + categoryFilter,+ query: baseQuery + categoryFilter + includeFilter + excludeFilter,categoryMap,};}@@ -135,7 +145,7 @@ export function getCategoryFromUrl(// Check against category map for other sitesfor (const [site, category] of categoryMap.entries()) {if (site === "__pdf__") continue; // Skip the special PDF marker-+if (hostname === site.toLowerCase() ||hostname.endsWith("." + site.toLowerCase())diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 3c94c904c..bf399f2af 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -26,6 +26,8 @@ interface SearchOptions {location?: string;sources: Array<{ type: string }>;categories?: CategoryOption[];+ includeDomains?: string[];+ excludeDomains?: string[];enterprise?: ("default" | "anon" | "zdr")[];scrapeOptions?: ScrapeOptions;timeout: number;@@ -79,6 +81,10 @@ export async function executeSearch(const { query: searchQuery, categoryMap } = buildSearchQuery(query,categories,+ {+ includeDomains: options.includeDomains,+ excludeDomains: options.excludeDomains,+ },);const searchResponse = (await search({diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.csindex a51ee10dd..4d44144b7 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs@@ -84,13 +84,17 @@ public class ModelsTests{Limit = 5,Location = "US",- Tbs = "qdr:w"+ Tbs = "qdr:w",+ IncludeDomains = new() { "firecrawl.dev" },+ ExcludeDomains = new() { "example.com" }};var json = JsonSerializer.Serialize(options, JsonOptions);Assert.Contains("\"limit\":5", json);Assert.Contains("\"location\":\"US\"", json);Assert.Contains("\"tbs\":\"qdr:w\"", json);+ Assert.Contains("\"includeDomains\":[\"firecrawl.dev\"]", json);+ Assert.Contains("\"excludeDomains\":[\"example.com\"]", json);}[Fact]diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex 4f3048fb1..6bd4e455a 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.1.1</Version>+ <Version>1.2.0</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.csindex 2cb411c51..dbc35cd19 100644--- a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs@@ -15,6 +15,14 @@ public class SearchOptions[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public List<object>? Categories { get; set; }+ [JsonPropertyName("includeDomains")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public List<string>? IncludeDomains { get; set; }++ [JsonPropertyName("excludeDomains")]+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]+ public List<string>? ExcludeDomains { get; set; }+[JsonPropertyName("limit")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]public int? Limit { get; set; }diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 5e8ba9ea5..6faaa94b5 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -1025,7 +1025,9 @@ defmodule Firecrawl docategories: [type: {:list, :any}, doc: "Categories to filter results by. Defaults to [], which means results will not be filtered by any categories."],country: [type: :string, doc: "ISO country code for geo-targeting search results (e.g. `US`). For best results, set both this and the `location` parameter."],enterprise: [type: {:list, :string}, doc: "Enterprise search options for Zero Data Retention (ZDR). Use `[\"zdr\"]` for end-to-end ZDR (10 credits / 10 results) or `[\"anon\"]` for anonymized ZDR (2 credits / 10 results). Must be enabled for your team."],+ exclude_domains: [type: {:list, :string}, doc: "Domains to exclude from search results."],ignore_invalid_urls: [type: :boolean, doc: "Excludes URLs from the search results that are invalid for other Firecrawl endpoints. This helps reduce errors if you are piping data from search into other Firecrawl API endpoints."],+ include_domains: [type: {:list, :string}, doc: "Domains to include in search results."],limit: [type: :integer, doc: "Maximum number of results to return"],location: [type: :string, doc: "Location parameter for search results (e.g. `San Francisco,California,United States`). For best results, set both this and the `country` parameter."],query: [type: :string, required: true, doc: "The search query"],@@ -1035,7 +1037,7 @@ defmodule Firecrawl dotimeout: [type: :integer, doc: "Timeout in milliseconds"]])- @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", ignore_invalid_urls: "ignoreInvalidURLs", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"}+ @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"}@doc """Search and optionally scrape search resultsdiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 17950d843..12db15dc3 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.1.0"+ @version "1.2.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.goindex a824be667..75766bc77 100644--- a/apps/go-sdk/options.go+++ b/apps/go-sdk/options.go@@ -74,6 +74,8 @@ type MapOptions struct {type SearchOptions struct {Sources []interface{} `json:"sources,omitempty"`Categories []interface{} `json:"categories,omitempty"`+ IncludeDomains []string `json:"includeDomains,omitempty"`+ ExcludeDomains []string `json:"excludeDomains,omitempty"`Limit *int `json:"limit,omitempty"`TBS *string `json:"tbs,omitempty"`Location *string `json:"location,omitempty"`diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 8f8222c21..e0d6e24e1 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.1.1"+const Version = "1.2.0"diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.javaindex c58447f53..8b36091f8 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java@@ -11,6 +11,8 @@ public class SearchOptions {private List<Object> sources;private List<Object> categories;+ private List<String> includeDomains;+ private List<String> excludeDomains;private Integer limit;private String tbs;private String location;@@ -23,6 +25,8 @@ public class SearchOptions {public List<Object> getSources() { return sources; }public List<Object> getCategories() { return categories; }+ public List<String> getIncludeDomains() { return includeDomains; }+ public List<String> getExcludeDomains() { return excludeDomains; }public Integer getLimit() { return limit; }public String getTbs() { return tbs; }public String getLocation() { return location; }@@ -36,6 +40,8 @@ public class SearchOptions {public static final class Builder {private List<Object> sources;private List<Object> categories;+ private List<String> includeDomains;+ private List<String> excludeDomains;private Integer limit;private String tbs;private String location;@@ -50,6 +56,10 @@ public class SearchOptions {public Builder sources(List<Object> sources) { this.sources = sources; return this; }/** Categories: "github", "research", "pdf". */public Builder categories(List<Object> categories) { this.categories = categories; return this; }+ /** Domains to include in search results. */+ public Builder includeDomains(List<String> includeDomains) { this.includeDomains = includeDomains; return this; }+ /** Domains to exclude from search results. */+ public Builder excludeDomains(List<String> excludeDomains) { this.excludeDomains = excludeDomains; return this; }/** Maximum number of results. */public Builder limit(Integer limit) { this.limit = limit; return this; }/** Time-based search filter (e.g., "qdr:d" for past day, "qdr:w" for past week). */@@ -69,6 +79,8 @@ public class SearchOptions {SearchOptions o = new SearchOptions();o.sources = this.sources;o.categories = this.categories;+ o.includeDomains = this.includeDomains;+ o.excludeDomains = this.excludeDomains;o.limit = this.limit;o.tbs = this.tbs;o.location = this.location;diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex ef5a31e4b..c8b729bdf 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.20.1",+ "version": "4.21.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.tsindex 3abf53085..0d1b54b8c 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts@@ -20,11 +20,17 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {throw new Error("limit must be positive");if (req.timeout != null && req.timeout <= 0)throw new Error("timeout must be positive");+ if (req.includeDomains?.length && req.excludeDomains?.length)+ throw new Error(+ "includeDomains and excludeDomains cannot both be specified",+ );const payload: Record<string, unknown> = {query: req.query,};if (req.sources) payload.sources = req.sources;if (req.categories) payload.categories = req.categories;+ if (req.includeDomains) payload.includeDomains = req.includeDomains;+ if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;if (req.limit != null) payload.limit = req.limit;if (req.tbs != null) payload.tbs = req.tbs;if (req.location != null) payload.location = req.location;diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 51ce6c262..8a3fa0334 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -508,6 +508,8 @@ export interface SearchRequest {'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }>;categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;+ includeDomains?: string[];+ excludeDomains?: string[];limit?: number;tbs?: string;location?: string;diff --git a/apps/php-sdk/src/Models/SearchOptions.php b/apps/php-sdk/src/Models/SearchOptions.phpindex 077a7b50b..30dd484fd 100644--- a/apps/php-sdk/src/Models/SearchOptions.php+++ b/apps/php-sdk/src/Models/SearchOptions.php@@ -9,6 +9,8 @@ final class SearchOptions/*** @param list<mixed>|null $sources* @param list<mixed>|null $categories+ * @param list<string>|null $includeDomains+ * @param list<string>|null $excludeDomains*/private function __construct(private readonly ?array $sources = null,@@ -20,11 +22,15 @@ final class SearchOptionsprivate readonly ?int $timeout = null,private readonly ?ScrapeOptions $scrapeOptions = null,private readonly ?string $integration = null,+ private readonly ?array $includeDomains = null,+ private readonly ?array $excludeDomains = null,) {}/*** @param list<mixed>|null $sources* @param list<mixed>|null $categories+ * @param list<string>|null $includeDomains+ * @param list<string>|null $excludeDomains*/public static function with(?array $sources = null,@@ -36,10 +42,12 @@ final class SearchOptions?int $timeout = null,?ScrapeOptions $scrapeOptions = null,?string $integration = null,+ ?array $includeDomains = null,+ ?array $excludeDomains = null,): self {return new self(- $sources, $categories, $limit, $tbs, $location,- $ignoreInvalidURLs, $timeout, $scrapeOptions, $integration,+ $sources, $categories, $limit, $tbs, $location, $ignoreInvalidURLs,+ $timeout, $scrapeOptions, $integration, $includeDomains, $excludeDomains,);}@@ -49,6 +57,8 @@ final class SearchOptions$fields = ['sources' => $this->sources,'categories' => $this->categories,+ 'includeDomains' => $this->includeDomains,+ 'excludeDomains' => $this->excludeDomains,'limit' => $this->limit,'tbs' => $this->tbs,'location' => $this->location,diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 8e20c7177..492fe6f85 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.23.1"+__version__ = "4.24.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.pyindex 8468edb5f..bddf6dc23 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py@@ -27,6 +27,7 @@ class TestAsyncSearchRequestPreparation:request = SearchRequest(query="test query",sources=["web", "news"],+ exclude_domains=["example.com"],limit=10,tbs="qdr:w",location="US",@@ -37,6 +38,8 @@ class TestAsyncSearchRequestPreparation:)data = _prepare_search_request(request)assert data["ignoreInvalidURLs"] is False+ assert data["excludeDomains"] == ["example.com"]+ assert "exclude_domains" not in dataassert "scrapeOptions" in dataassert data["integration"] == "_unit-test"@@ -55,6 +58,17 @@ class TestAsyncSearchRequestPreparation:assert "query" in dataassert len(data) == 1+ def test_domain_filters_are_mutually_exclusive(self):+ with pytest.raises(+ ValueError,+ match="include_domains and exclude_domains cannot both be specified",+ ):+ SearchRequest(+ query="test",+ include_domains=["firecrawl.dev"],+ exclude_domains=["example.com"],+ )+def test_empty_scrape_options(self):request = SearchRequest(query="test", scrape_options=ScrapeOptions())data = _prepare_search_request(request)diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.pyindex a2aebef42..f63219591 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py@@ -38,6 +38,7 @@ class TestSearchRequestPreparation:request = SearchRequest(query="test query",sources=["web", "news"],+ include_domains=["firecrawl.dev"],limit=10,tbs="qdr:w",location="US",@@ -69,6 +70,8 @@ class TestSearchRequestPreparation:assert len(data["sources"]) == 2assert data["sources"][0]["type"] == "web"assert data["sources"][1]["type"] == "news"+ assert data["includeDomains"] == ["firecrawl.dev"]+ assert "include_domains" not in data# Check nested scrape options conversionsscrape_data = data["scrapeOptions"]@@ -105,6 +108,18 @@ class TestSearchRequestPreparation:assert "query" in dataassert len(data) == 1 # Only query should be present+ def test_domain_filters_are_mutually_exclusive(self):+ """Test that include_domains and exclude_domains cannot both be set."""+ with pytest.raises(+ ValueError,+ match="include_domains and exclude_domains cannot both be specified",+ ):+ SearchRequest(+ query="test",+ include_domains=["firecrawl.dev"],+ exclude_domains=["example.com"],+ )+def test_empty_scrape_options(self):"""Test that empty scrape options are handled correctly."""scrape_opts = ScrapeOptions() # All defaultsdiff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.pyindex b32e6dabe..54c5e0d39 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py@@ -118,6 +118,11 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:if source.type not in valid_sources:raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")+ if request.include_domains and request.exclude_domains:+ raise ValueError(+ "include_domains and exclude_domains cannot both be specified"+ )+if request.location is not None:if not isinstance(request.location, str) or len(request.location.strip()) == 0:raise ValueError("Location must be a non-empty string")@@ -153,6 +158,14 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urlsdata.pop("ignore_invalid_urls", None)+ if validated_request.include_domains is not None:+ data["includeDomains"] = validated_request.include_domains+ data.pop("include_domains", None)++ if validated_request.exclude_domains is not None:+ data["excludeDomains"] = validated_request.exclude_domains+ data.pop("exclude_domains", None)+if validated_request.scrape_options is not None:scrape_data = prepare_scrape_options(validated_request.scrape_options)if scrape_data:diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 733695f01..118496f09 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -141,6 +141,11 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:elif hasattr(category, 'type'):if category.type not in valid_categories:raise ValueError(f"Invalid category type: {category.type}. Valid types: {valid_categories}")++ if request.include_domains and request.exclude_domains:+ raise ValueError(+ "include_domains and exclude_domains cannot both be specified"+ )# Validate location (if provided)if request.location is not None:@@ -185,6 +190,16 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:if validated_request.ignore_invalid_urls is not None:data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urlsdata.pop("ignore_invalid_urls", None)++ # include_domains → includeDomains+ if validated_request.include_domains is not None:+ data["includeDomains"] = validated_request.include_domains+ data.pop("include_domains", None)++ # exclude_domains → excludeDomains+ if validated_request.exclude_domains is not None:+ data["excludeDomains"] = validated_request.exclude_domains+ data.pop("exclude_domains", None)# scrape_options → scrapeOptionsif validated_request.scrape_options is not None:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 50776997e..a8e9b6b06 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -1070,6 +1070,8 @@ class SearchRequest(BaseModel):query: strsources: Optional[List[SourceOption]] = Nonecategories: Optional[List[CategoryOption]] = None+ include_domains: Optional[List[str]] = None+ exclude_domains: Optional[List[str]] = Nonelimit: Optional[int] = 5tbs: Optional[str] = Nonelocation: Optional[str] = None@@ -1118,6 +1120,15 @@ class SearchRequest(BaseModel):return normalized_categories+ @model_validator(mode="after")+ def validate_domain_filters(self):+ """Validate mutually exclusive search domain filters."""+ if self.include_domains and self.exclude_domains:+ raise ValueError(+ "include_domains and exclude_domains cannot both be specified"+ )+ return self+# NOTE: parsers validation does not belong on SearchRequest; it is part of ScrapeOptions.diff --git a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb b/apps/ruby-sdk/lib/firecrawl/models/search_options.rbindex b3d2d5f39..da05a5c0d 100644--- a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb@@ -5,8 +5,8 @@ module Firecrawl# Options for a web search request.class SearchOptionsFIELDS = %i[- sources categories limit tbs location ignore_invalid_urls- timeout scrape_options integration+ sources categories include_domains exclude_domains limit tbs location+ ignore_invalid_urls timeout scrape_options integration].freezeattr_reader(*FIELDS)@@ -19,6 +19,8 @@ module Firecrawl{"sources" => sources,"categories" => categories,+ "includeDomains" => include_domains,+ "excludeDomains" => exclude_domains,"limit" => limit,"tbs" => tbs,"location" => location,diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex 3fdad5687..5b4547f4a 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -500,12 +500,16 @@ class ClientTest < Minitest::Testopts = Firecrawl::Models::SearchOptions.new(limit: 10,location: "US",- tbs: "qdr:w"+ tbs: "qdr:w",+ include_domains: ["firecrawl.dev"],+ exclude_domains: ["example.com"])h = opts.to_hassert_equal 10, h["limit"]assert_equal "US", h["location"]assert_equal "qdr:w", h["tbs"]+ assert_equal ["firecrawl.dev"], h["includeDomains"]+ assert_equal ["example.com"], h["excludeDomains"]enddef test_agent_options_to_hdiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex ce006108a..7f17f6a56 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.1.0"+version = "2.3.0"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex ee65270a9..7bb9bf1b6 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.2.0"+version = "2.3.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rsindex 534559bbb..a6d58436f 100644--- a/apps/rust-sdk/src/search.rs+++ b/apps/rust-sdk/src/search.rs@@ -23,6 +23,12 @@ pub struct SearchOptions {/// Categories to filter results (github, research, pdf).pub categories: Option<Vec<SearchCategory>>,+ /// Domains to include in search results.+ pub include_domains: Option<Vec<String>>,++ /// Domains to exclude from search results.+ pub exclude_domains: Option<Vec<String>>,+/// Time-based search filter (e.g., "qdr:d" for past day).pub tbs: Option<String>,