Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): includeDomains/excludeDomains (#3466)

Gergő Móricz105fc997bc30
diff --git a/apps/api/src/__tests__/lib/search-query-builder.test.ts b/apps/api/src/__tests__/lib/search-query-builder.test.ts
index 8da3d02f0..f746dd2c2 100644
--- a/apps/api/src/__tests__/lib/search-query-builder.test.ts
+++ b/apps/api/src/__tests__/lib/search-query-builder.test.ts
@@ -110,6 +110,45 @@ describe("Search Query Builder", () => {
expect(result.query).toBe("test");
expect(result.categoryMap.size).toBe(0);
});
+
+ it("should add include domain filters", () => {
+ const result = buildSearchQuery("web scraping", undefined, {
+ includeDomains: ["firecrawl.dev", "docs.firecrawl.dev"],
+ });
+ expect(result.query).toBe(
+ "web scraping (site:firecrawl.dev OR site:docs.firecrawl.dev)",
+ );
+ expect(result.categoryMap.size).toBe(0);
+ });
+
+ it("should add exclude domain filters", () => {
+ const result = buildSearchQuery("web scraping", undefined, {
+ excludeDomains: ["example.com", "spam.example.com"],
+ });
+ expect(result.query).toBe(
+ "web scraping -site:example.com -site:spam.example.com",
+ );
+ expect(result.categoryMap.size).toBe(0);
+ });
+
+ it("should ignore empty domain filter arrays", () => {
+ const result = buildSearchQuery("web scraping", undefined, {
+ includeDomains: [],
+ excludeDomains: [],
+ });
+ expect(result.query).toBe("web scraping");
+ expect(result.categoryMap.size).toBe(0);
+ });
+
+ it("should combine categories with domain filters", () => {
+ const result = buildSearchQuery("web scraping", ["github"], {
+ includeDomains: ["firecrawl.dev"],
+ });
+ expect(result.query).toBe(
+ "web scraping (site:github.com) (site:firecrawl.dev)",
+ );
+ expect(result.categoryMap.get("github.com")).toBe("github");
+ });
});
describe("getCategoryFromUrl", () => {
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 78c54f3b2..840a2c7c7 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -478,7 +478,7 @@ export function expectMapToSucceed(response: Awaited<ReturnType<typeof map>>) {
// Search API
// =========================================
-async function searchRaw(body: SearchRequestInput, identity: Identity) {
+export async function searchRaw(body: SearchRequestInput, identity: Identity) {
return await request(TEST_API_URL)
.post("/v2/search")
.set("Authorization", `Bearer ${identity.apiKey}`)
@@ -494,6 +494,12 @@ function expectSearchToSucceed(
expect(typeof response.body.data).toBe("object");
}
+function expectSearchToFail(response: Awaited<ReturnType<typeof searchRaw>>) {
+ expect(response.statusCode).not.toBe(200);
+ expect(response.body.success).toBe(false);
+ expect(typeof response.body.error).toBe("string");
+}
+
export async function search(
body: SearchRequestInput,
identity: Identity,
@@ -503,6 +509,19 @@ export async function search(
return raw.body.data;
}
+export async function searchWithFailure(
+ body: SearchRequestInput,
+ identity: Identity,
+): Promise<{
+ success: false;
+ error: string;
+ details?: unknown;
+}> {
+ const raw = await searchRaw(body, identity);
+ expectSearchToFail(raw);
+ return raw.body;
+}
+
// =========================================
// Billing API
// =========================================
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index b91ab1551..60fdc5658 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -5,7 +5,7 @@ import {
HAS_SEARCH,
TEST_PRODUCTION,
} from "../lib";
-import { search, idmux, Identity } from "./lib";
+import { search, searchWithFailure, idmux, Identity } from "./lib";
import { config } from "../../../config";
let identity: Identity;
@@ -35,6 +35,48 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {
60000,
);
+ it.concurrent(
+ "works with includeDomains",
+ async () => {
+ const res = await search(
+ {
+ query: "firecrawl",
+ includeDomains: ["firecrawl.dev"],
+ limit: 3,
+ },
+ identity,
+ );
+
+ expect(res.web).toBeDefined();
+ expect(res.web?.length).toBeGreaterThan(0);
+ for (const result of res.web ?? []) {
+ expect(result.url).toBeDefined();
+ const hostname = new URL(result.url!).hostname;
+ expect(
+ hostname === "firecrawl.dev" || hostname.endsWith(".firecrawl.dev"),
+ ).toBe(true);
+ }
+ },
+ 60000,
+ );
+
+ it.concurrent(
+ "rejects includeDomains with excludeDomains",
+ async () => {
+ const res = await searchWithFailure(
+ {
+ query: "firecrawl",
+ includeDomains: ["firecrawl.dev"],
+ excludeDomains: ["example.com"],
+ },
+ identity,
+ );
+
+ expect(res.error).toBe("Invalid request body");
+ },
+ 60000,
+ );
+
it.concurrent(
"works with scrape",
async () => {
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 98c91032d..4e97a5731 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -914,6 +914,44 @@ describe("V2 Types Validation", () => {
expect(Array.isArray(result.categories)).toBe(true);
});
+ it("should accept search request with includeDomains", () => {
+ const input: SearchRequestInput = {
+ query: "test",
+ includeDomains: [" Example.com ", "docs.example.com"],
+ };
+
+ const result = searchRequestSchema.parse(input);
+ expect(result.includeDomains).toEqual([
+ "example.com",
+ "docs.example.com",
+ ]);
+ });
+
+ it("should accept search request with excludeDomains", () => {
+ const input: SearchRequestInput = {
+ query: "test",
+ excludeDomains: ["example.com", "spam.example.com"],
+ };
+
+ const result = searchRequestSchema.parse(input);
+ expect(result.excludeDomains).toEqual([
+ "example.com",
+ "spam.example.com",
+ ]);
+ });
+
+ it("should reject search request with includeDomains and excludeDomains", () => {
+ const input: SearchRequestInput = {
+ query: "test",
+ includeDomains: ["example.com"],
+ excludeDomains: ["spam.example.com"],
+ };
+
+ expect(() => searchRequestSchema.parse(input)).toThrow(
+ "includeDomains and excludeDomains cannot both be specified",
+ );
+ });
+
it("should reject limit exceeding 100", () => {
const input: SearchRequestInput = {
query: "test",
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 74b87b2c8..74160e63e 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -127,6 +127,8 @@ export async function searchController(
location: req.body.location,
sources: req.body.sources as Array<{ type: string }>,
categories: req.body.categories as CategoryOption[],
+ includeDomains: req.body.includeDomains,
+ excludeDomains: req.body.excludeDomains,
enterprise: req.body.enterprise,
scrapeOptions: req.body.scrapeOptions,
timeout: req.body.timeout,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 961c123f7..f11bb094c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1703,6 +1703,18 @@ const pdfCategoryOptions = z.strictObject({
type: z.literal("pdf"),
});
+const searchDomainSchema = z
+ .string()
+ .trim()
+ .toLowerCase()
+ .refine(
+ value =>
+ /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/.test(
+ value,
+ ),
+ "Domain must be a valid hostname without protocol or path",
+ );
+
export const searchRequestSchema = z
.strictObject({
query: z.string(),
@@ -1738,6 +1750,8 @@ export const searchRequestSchema = z
),
])
.optional(),
+ includeDomains: z.array(searchDomainSchema).optional(),
+ excludeDomains: z.array(searchDomainSchema).optional(),
lang: z.string().optional().prefault("en"),
enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional(),
country: z.string().optional(),
@@ -1790,6 +1804,10 @@ export const searchRequestSchema = z
})
.optional(),
})
+ .refine(
+ x => !(x.includeDomains?.length && x.excludeDomains?.length),
+ "includeDomains and excludeDomains cannot both be specified",
+ )
.refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform(x => {
const country =
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index bcaaf2eb6..098ac2ae8 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -284,6 +284,10 @@ export async function x402SearchController(
const { query: searchQuery, categoryMap } = buildSearchQuery(
req.body.query,
req.body.categories as CategoryOption[],
+ {
+ includeDomains: req.body.includeDomains,
+ excludeDomains: req.body.excludeDomains,
+ },
);
const searchResponse = (await search({
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index 9619d1ec8..8f1b2797b 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -10,6 +10,11 @@ interface CategoryInput {
export type CategoryOption = string | CategoryInput;
+interface DomainFilterOptions {
+ includeDomains?: string[];
+ excludeDomains?: string[];
+}
+
interface QueryBuilderResult {
query: string;
categoryMap: Map<string, string>;
@@ -42,20 +47,14 @@ const DEFAULT_RESEARCH_SITES = [
export function buildSearchQuery(
baseQuery: string,
categories?: CategoryOption[],
+ domainOptions: DomainFilterOptions = {},
): QueryBuilderResult {
const categoryMap = new Map<string, string>();
- if (!categories || categories.length === 0) {
- return {
- query: baseQuery,
- categoryMap,
- };
- }
-
const siteFilters: string[] = [];
let hasPdfFilter = false;
- for (const category of categories) {
+ for (const category of categories ?? []) {
if (typeof category === "string") {
// Simple string format
if (category === "github") {
@@ -101,8 +100,19 @@ export function buildSearchQuery(
categoryFilter += " filetype:pdf";
}
+ const includeDomains = domainOptions.includeDomains ?? [];
+ const excludeDomains = domainOptions.excludeDomains ?? [];
+ const includeFilter =
+ includeDomains.length > 0
+ ? " (" + includeDomains.map(domain => `site:${domain}`).join(" OR ") + ")"
+ : "";
+ const excludeFilter =
+ excludeDomains.length > 0
+ ? " " + excludeDomains.map(domain => `-site:${domain}`).join(" ")
+ : "";
+
return {
- query: baseQuery + categoryFilter,
+ query: baseQuery + categoryFilter + includeFilter + excludeFilter,
categoryMap,
};
}
@@ -135,7 +145,7 @@ export function getCategoryFromUrl(
// Check against category map for other sites
for (const [site, category] of categoryMap.entries()) {
if (site === "__pdf__") continue; // Skip the special PDF marker
-
+
if (
hostname === site.toLowerCase() ||
hostname.endsWith("." + site.toLowerCase())
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 3c94c904c..bf399f2af 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -26,6 +26,8 @@ interface SearchOptions {
location?: string;
sources: Array<{ type: string }>;
categories?: CategoryOption[];
+ includeDomains?: string[];
+ excludeDomains?: string[];
enterprise?: ("default" | "anon" | "zdr")[];
scrapeOptions?: ScrapeOptions;
timeout: number;
@@ -79,6 +81,10 @@ export async function executeSearch(
const { query: searchQuery, categoryMap } = buildSearchQuery(
query,
categories,
+ {
+ includeDomains: options.includeDomains,
+ excludeDomains: options.excludeDomains,
+ },
);
const searchResponse = (await search({
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
index a51ee10dd..4d44144b7 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
@@ -84,13 +84,17 @@ public class ModelsTests
{
Limit = 5,
Location = "US",
- Tbs = "qdr:w"
+ Tbs = "qdr:w",
+ IncludeDomains = new() { "firecrawl.dev" },
+ ExcludeDomains = new() { "example.com" }
};
var json = JsonSerializer.Serialize(options, JsonOptions);
Assert.Contains("\"limit\":5", json);
Assert.Contains("\"location\":\"US\"", json);
Assert.Contains("\"tbs\":\"qdr:w\"", json);
+ Assert.Contains("\"includeDomains\":[\"firecrawl.dev\"]", json);
+ Assert.Contains("\"excludeDomains\":[\"example.com\"]", json);
}
[Fact]
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index 4f3048fb1..6bd4e455a 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.1.1</Version>
+ <Version>1.2.0</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs
index 2cb411c51..dbc35cd19 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/SearchOptions.cs
@@ -15,6 +15,14 @@ public class SearchOptions
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public List<object>? Categories { get; set; }
+ [JsonPropertyName("includeDomains")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public List<string>? IncludeDomains { get; set; }
+
+ [JsonPropertyName("excludeDomains")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public List<string>? ExcludeDomains { get; set; }
+
[JsonPropertyName("limit")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? Limit { get; set; }
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 5e8ba9ea5..6faaa94b5 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -1025,7 +1025,9 @@ defmodule Firecrawl do
categories: [type: {:list, :any}, doc: "Categories to filter results by. Defaults to [], which means results will not be filtered by any categories."],
country: [type: :string, doc: "ISO country code for geo-targeting search results (e.g. `US`). For best results, set both this and the `location` parameter."],
enterprise: [type: {:list, :string}, doc: "Enterprise search options for Zero Data Retention (ZDR). Use `[\"zdr\"]` for end-to-end ZDR (10 credits / 10 results) or `[\"anon\"]` for anonymized ZDR (2 credits / 10 results). Must be enabled for your team."],
+ exclude_domains: [type: {:list, :string}, doc: "Domains to exclude from search results."],
ignore_invalid_urls: [type: :boolean, doc: "Excludes URLs from the search results that are invalid for other Firecrawl endpoints. This helps reduce errors if you are piping data from search into other Firecrawl API endpoints."],
+ include_domains: [type: {:list, :string}, doc: "Domains to include in search results."],
limit: [type: :integer, doc: "Maximum number of results to return"],
location: [type: :string, doc: "Location parameter for search results (e.g. `San Francisco,California,United States`). For best results, set both this and the `country` parameter."],
query: [type: :string, required: true, doc: "The search query"],
@@ -1035,7 +1037,7 @@ defmodule Firecrawl do
timeout: [type: :integer, doc: "Timeout in milliseconds"]
])
- @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", ignore_invalid_urls: "ignoreInvalidURLs", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"}
+ @search_and_scrape_key_mapping %{categories: "categories", country: "country", enterprise: "enterprise", exclude_domains: "excludeDomains", ignore_invalid_urls: "ignoreInvalidURLs", include_domains: "includeDomains", limit: "limit", location: "location", query: "query", scrape_options: "scrapeOptions", sources: "sources", tbs: "tbs", timeout: "timeout"}
@doc """
Search and optionally scrape search results
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 17950d843..12db15dc3 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.1.0"
+ @version "1.2.0"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go
index a824be667..75766bc77 100644
--- a/apps/go-sdk/options.go
+++ b/apps/go-sdk/options.go
@@ -74,6 +74,8 @@ type MapOptions struct {
type SearchOptions struct {
Sources []interface{} `json:"sources,omitempty"`
Categories []interface{} `json:"categories,omitempty"`
+ IncludeDomains []string `json:"includeDomains,omitempty"`
+ ExcludeDomains []string `json:"excludeDomains,omitempty"`
Limit *int `json:"limit,omitempty"`
TBS *string `json:"tbs,omitempty"`
Location *string `json:"location,omitempty"`
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index 8f8222c21..e0d6e24e1 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.1.1"
+const Version = "1.2.0"
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java
index c58447f53..8b36091f8 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java
@@ -11,6 +11,8 @@ public class SearchOptions {
private List<Object> sources;
private List<Object> categories;
+ private List<String> includeDomains;
+ private List<String> excludeDomains;
private Integer limit;
private String tbs;
private String location;
@@ -23,6 +25,8 @@ public class SearchOptions {
public List<Object> getSources() { return sources; }
public List<Object> getCategories() { return categories; }
+ public List<String> getIncludeDomains() { return includeDomains; }
+ public List<String> getExcludeDomains() { return excludeDomains; }
public Integer getLimit() { return limit; }
public String getTbs() { return tbs; }
public String getLocation() { return location; }
@@ -36,6 +40,8 @@ public class SearchOptions {
public static final class Builder {
private List<Object> sources;
private List<Object> categories;
+ private List<String> includeDomains;
+ private List<String> excludeDomains;
private Integer limit;
private String tbs;
private String location;
@@ -50,6 +56,10 @@ public class SearchOptions {
public Builder sources(List<Object> sources) { this.sources = sources; return this; }
/** Categories: "github", "research", "pdf". */
public Builder categories(List<Object> categories) { this.categories = categories; return this; }
+ /** Domains to include in search results. */
+ public Builder includeDomains(List<String> includeDomains) { this.includeDomains = includeDomains; return this; }
+ /** Domains to exclude from search results. */
+ public Builder excludeDomains(List<String> excludeDomains) { this.excludeDomains = excludeDomains; return this; }
/** Maximum number of results. */
public Builder limit(Integer limit) { this.limit = limit; return this; }
/** Time-based search filter (e.g., "qdr:d" for past day, "qdr:w" for past week). */
@@ -69,6 +79,8 @@ public class SearchOptions {
SearchOptions o = new SearchOptions();
o.sources = this.sources;
o.categories = this.categories;
+ o.includeDomains = this.includeDomains;
+ o.excludeDomains = this.excludeDomains;
o.limit = this.limit;
o.tbs = this.tbs;
o.location = this.location;
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index ef5a31e4b..c8b729bdf 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.20.1",
+ "version": "4.21.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 3abf53085..0d1b54b8c 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -20,11 +20,17 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
throw new Error("limit must be positive");
if (req.timeout != null && req.timeout <= 0)
throw new Error("timeout must be positive");
+ if (req.includeDomains?.length && req.excludeDomains?.length)
+ throw new Error(
+ "includeDomains and excludeDomains cannot both be specified",
+ );
const payload: Record<string, unknown> = {
query: req.query,
};
if (req.sources) payload.sources = req.sources;
if (req.categories) payload.categories = req.categories;
+ if (req.includeDomains) payload.includeDomains = req.includeDomains;
+ if (req.excludeDomains) payload.excludeDomains = req.excludeDomains;
if (req.limit != null) payload.limit = req.limit;
if (req.tbs != null) payload.tbs = req.tbs;
if (req.location != null) payload.location = req.location;
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 51ce6c262..8a3fa0334 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -508,6 +508,8 @@ export interface SearchRequest {
'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }
>;
categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;
+ includeDomains?: string[];
+ excludeDomains?: string[];
limit?: number;
tbs?: string;
location?: string;
diff --git a/apps/php-sdk/src/Models/SearchOptions.php b/apps/php-sdk/src/Models/SearchOptions.php
index 077a7b50b..30dd484fd 100644
--- a/apps/php-sdk/src/Models/SearchOptions.php
+++ b/apps/php-sdk/src/Models/SearchOptions.php
@@ -9,6 +9,8 @@ final class SearchOptions
/**
* @param list<mixed>|null $sources
* @param list<mixed>|null $categories
+ * @param list<string>|null $includeDomains
+ * @param list<string>|null $excludeDomains
*/
private function __construct(
private readonly ?array $sources = null,
@@ -20,11 +22,15 @@ final class SearchOptions
private readonly ?int $timeout = null,
private readonly ?ScrapeOptions $scrapeOptions = null,
private readonly ?string $integration = null,
+ private readonly ?array $includeDomains = null,
+ private readonly ?array $excludeDomains = null,
) {}
/**
* @param list<mixed>|null $sources
* @param list<mixed>|null $categories
+ * @param list<string>|null $includeDomains
+ * @param list<string>|null $excludeDomains
*/
public static function with(
?array $sources = null,
@@ -36,10 +42,12 @@ final class SearchOptions
?int $timeout = null,
?ScrapeOptions $scrapeOptions = null,
?string $integration = null,
+ ?array $includeDomains = null,
+ ?array $excludeDomains = null,
): self {
return new self(
- $sources, $categories, $limit, $tbs, $location,
- $ignoreInvalidURLs, $timeout, $scrapeOptions, $integration,
+ $sources, $categories, $limit, $tbs, $location, $ignoreInvalidURLs,
+ $timeout, $scrapeOptions, $integration, $includeDomains, $excludeDomains,
);
}
@@ -49,6 +57,8 @@ final class SearchOptions
$fields = [
'sources' => $this->sources,
'categories' => $this->categories,
+ 'includeDomains' => $this->includeDomains,
+ 'excludeDomains' => $this->excludeDomains,
'limit' => $this->limit,
'tbs' => $this->tbs,
'location' => $this->location,
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 8e20c7177..492fe6f85 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.23.1"
+__version__ = "4.24.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py
index 8468edb5f..bddf6dc23 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_search_request_preparation.py
@@ -27,6 +27,7 @@ class TestAsyncSearchRequestPreparation:
request = SearchRequest(
query="test query",
sources=["web", "news"],
+ exclude_domains=["example.com"],
limit=10,
tbs="qdr:w",
location="US",
@@ -37,6 +38,8 @@ class TestAsyncSearchRequestPreparation:
)
data = _prepare_search_request(request)
assert data["ignoreInvalidURLs"] is False
+ assert data["excludeDomains"] == ["example.com"]
+ assert "exclude_domains" not in data
assert "scrapeOptions" in data
assert data["integration"] == "_unit-test"
@@ -55,6 +58,17 @@ class TestAsyncSearchRequestPreparation:
assert "query" in data
assert len(data) == 1
+ def test_domain_filters_are_mutually_exclusive(self):
+ with pytest.raises(
+ ValueError,
+ match="include_domains and exclude_domains cannot both be specified",
+ ):
+ SearchRequest(
+ query="test",
+ include_domains=["firecrawl.dev"],
+ exclude_domains=["example.com"],
+ )
+
def test_empty_scrape_options(self):
request = SearchRequest(query="test", scrape_options=ScrapeOptions())
data = _prepare_search_request(request)
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py
index a2aebef42..f63219591 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_request_preparation.py
@@ -38,6 +38,7 @@ class TestSearchRequestPreparation:
request = SearchRequest(
query="test query",
sources=["web", "news"],
+ include_domains=["firecrawl.dev"],
limit=10,
tbs="qdr:w",
location="US",
@@ -69,6 +70,8 @@ class TestSearchRequestPreparation:
assert len(data["sources"]) == 2
assert data["sources"][0]["type"] == "web"
assert data["sources"][1]["type"] == "news"
+ assert data["includeDomains"] == ["firecrawl.dev"]
+ assert "include_domains" not in data
# Check nested scrape options conversions
scrape_data = data["scrapeOptions"]
@@ -105,6 +108,18 @@ class TestSearchRequestPreparation:
assert "query" in data
assert len(data) == 1 # Only query should be present
+ def test_domain_filters_are_mutually_exclusive(self):
+ """Test that include_domains and exclude_domains cannot both be set."""
+ with pytest.raises(
+ ValueError,
+ match="include_domains and exclude_domains cannot both be specified",
+ ):
+ SearchRequest(
+ query="test",
+ include_domains=["firecrawl.dev"],
+ exclude_domains=["example.com"],
+ )
+
def test_empty_scrape_options(self):
"""Test that empty scrape options are handled correctly."""
scrape_opts = ScrapeOptions() # All defaults
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
index b32e6dabe..54c5e0d39 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
@@ -118,6 +118,11 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
if source.type not in valid_sources:
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
+ if request.include_domains and request.exclude_domains:
+ raise ValueError(
+ "include_domains and exclude_domains cannot both be specified"
+ )
+
if request.location is not None:
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
raise ValueError("Location must be a non-empty string")
@@ -153,6 +158,14 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
data.pop("ignore_invalid_urls", None)
+ if validated_request.include_domains is not None:
+ data["includeDomains"] = validated_request.include_domains
+ data.pop("include_domains", None)
+
+ if validated_request.exclude_domains is not None:
+ data["excludeDomains"] = validated_request.exclude_domains
+ data.pop("exclude_domains", None)
+
if validated_request.scrape_options is not None:
scrape_data = prepare_scrape_options(validated_request.scrape_options)
if scrape_data:
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index 733695f01..118496f09 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -141,6 +141,11 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
elif hasattr(category, 'type'):
if category.type not in valid_categories:
raise ValueError(f"Invalid category type: {category.type}. Valid types: {valid_categories}")
+
+ if request.include_domains and request.exclude_domains:
+ raise ValueError(
+ "include_domains and exclude_domains cannot both be specified"
+ )
# Validate location (if provided)
if request.location is not None:
@@ -185,6 +190,16 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
if validated_request.ignore_invalid_urls is not None:
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
data.pop("ignore_invalid_urls", None)
+
+ # include_domains → includeDomains
+ if validated_request.include_domains is not None:
+ data["includeDomains"] = validated_request.include_domains
+ data.pop("include_domains", None)
+
+ # exclude_domains → excludeDomains
+ if validated_request.exclude_domains is not None:
+ data["excludeDomains"] = validated_request.exclude_domains
+ data.pop("exclude_domains", None)
# scrape_options → scrapeOptions
if validated_request.scrape_options is not None:
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 50776997e..a8e9b6b06 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -1070,6 +1070,8 @@ class SearchRequest(BaseModel):
query: str
sources: Optional[List[SourceOption]] = None
categories: Optional[List[CategoryOption]] = None
+ include_domains: Optional[List[str]] = None
+ exclude_domains: Optional[List[str]] = None
limit: Optional[int] = 5
tbs: Optional[str] = None
location: Optional[str] = None
@@ -1118,6 +1120,15 @@ class SearchRequest(BaseModel):
return normalized_categories
+ @model_validator(mode="after")
+ def validate_domain_filters(self):
+ """Validate mutually exclusive search domain filters."""
+ if self.include_domains and self.exclude_domains:
+ raise ValueError(
+ "include_domains and exclude_domains cannot both be specified"
+ )
+ return self
+
# NOTE: parsers validation does not belong on SearchRequest; it is part of ScrapeOptions.
diff --git a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb
index b3d2d5f39..da05a5c0d 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/search_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/search_options.rb
@@ -5,8 +5,8 @@ module Firecrawl
# Options for a web search request.
class SearchOptions
FIELDS = %i[
- sources categories limit tbs location ignore_invalid_urls
- timeout scrape_options integration
+ sources categories include_domains exclude_domains limit tbs location
+ ignore_invalid_urls timeout scrape_options integration
].freeze
attr_reader(*FIELDS)
@@ -19,6 +19,8 @@ module Firecrawl
{
"sources" => sources,
"categories" => categories,
+ "includeDomains" => include_domains,
+ "excludeDomains" => exclude_domains,
"limit" => limit,
"tbs" => tbs,
"location" => location,
diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb
index 3fdad5687..5b4547f4a 100644
--- a/apps/ruby-sdk/test/firecrawl/client_test.rb
+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb
@@ -500,12 +500,16 @@ class ClientTest < Minitest::Test
opts = Firecrawl::Models::SearchOptions.new(
limit: 10,
location: "US",
- tbs: "qdr:w"
+ tbs: "qdr:w",
+ include_domains: ["firecrawl.dev"],
+ exclude_domains: ["example.com"]
)
h = opts.to_h
assert_equal 10, h["limit"]
assert_equal "US", h["location"]
assert_equal "qdr:w", h["tbs"]
+ assert_equal ["firecrawl.dev"], h["includeDomains"]
+ assert_equal ["example.com"], h["excludeDomains"]
end
def test_agent_options_to_h
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index ce006108a..7f17f6a56 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.1.0"
+version = "2.3.0"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index ee65270a9..7bb9bf1b6 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.2.0"
+version = "2.3.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs
index 534559bbb..a6d58436f 100644
--- a/apps/rust-sdk/src/search.rs
+++ b/apps/rust-sdk/src/search.rs
@@ -23,6 +23,12 @@ pub struct SearchOptions {
/// Categories to filter results (github, research, pdf).
pub categories: Option<Vec<SearchCategory>>,
+ /// Domains to include in search results.
+ pub include_domains: Option<Vec<String>>,
+
+ /// Domains to exclude from search results.
+ pub exclude_domains: Option<Vec<String>>,
+
/// Time-based search filter (e.g., "qdr:d" for past day).
pub tbs: Option<String>,