Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: pdf search category
Nicolas8a3936fdc0d5
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex deb59f7ec..d9becd702 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1413,6 +1413,12 @@ const researchCategoryOptions = z}).strict();+const pdfCategoryOptions = z+ .object({+ type: z.literal("pdf"),+ })+ .strict();+export const searchRequestSchema = z.object({query: z.string(),@@ -1445,9 +1451,9 @@ export const searchRequestSchema = zcategories: z.union([// Array of strings (simple format)- z.array(z.enum(["github", "research"])),+ z.array(z.enum(["github", "research", "pdf"])),// Array of objects (advanced format)- z.array(z.union([githubCategoryOptions, researchCategoryOptions])),+ z.array(z.union([githubCategoryOptions, researchCategoryOptions, pdfCategoryOptions])),]).optional(),lang: z.string().optional().default("en"),@@ -1552,6 +1558,10 @@ export const searchRequestSchema = zreturn {type: "research" as const,};+ case "pdf":+ return {+ type: "pdf" as const,+ };default:return { type: c as any };}diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex 613d0fd86..9619d1ec8 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -4,7 +4,7 @@*/interface CategoryInput {- type: "github" | "research";+ type: "github" | "research" | "pdf";sites?: string[];}@@ -53,6 +53,7 @@ export function buildSearchQuery(}const siteFilters: string[] = [];+ let hasPdfFilter = false;for (const category of categories) {if (typeof category === "string") {@@ -66,6 +67,9 @@ export function buildSearchQuery(siteFilters.push(`site:${site}`);categoryMap.set(site, "research");}+ } else if (category === "pdf") {+ hasPdfFilter = true;+ categoryMap.set("__pdf__", "pdf");}} else {// Object format with options@@ -79,6 +83,9 @@ export function buildSearchQuery(siteFilters.push(`site:${site}`);categoryMap.set(site, "research");}+ } else if (category.type === "pdf") {+ hasPdfFilter = true;+ categoryMap.set("__pdf__", "pdf");}}}@@ -89,6 +96,11 @@ export function buildSearchQuery(categoryFilter = " (" + siteFilters.join(" OR ") + ")";}+ // Add filetype:pdf filter if PDF category is requested+ if (hasPdfFilter) {+ categoryFilter += " filetype:pdf";+ }+return {query: baseQuery + categoryFilter,categoryMap,@@ -108,6 +120,12 @@ export function getCategoryFromUrl(try {const urlObj = new URL(url);const hostname = urlObj.hostname.toLowerCase();+ const pathname = urlObj.pathname.toLowerCase();++ // Check if URL points to a PDF file+ if (pathname.endsWith(".pdf") && categoryMap.has("__pdf__")) {+ return "pdf";+ }// Direct match for GitHubif (hostname === "github.com" || hostname.endsWith(".github.com")) {@@ -116,6 +134,8 @@ export function getCategoryFromUrl(// Check against category map for other sitesfor (const [site, category] of categoryMap.entries()) {+ if (site === "__pdf__") continue; // Skip the special PDF marker+if (hostname === site.toLowerCase() ||hostname.endsWith("." + site.toLowerCase())diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex a6b3de587..866315f5f 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.3.8",+ "version": "4.4.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex dbb142581..13c58cd0e 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -286,7 +286,7 @@ export interface SearchData {}export interface CategoryOption {- type: 'github' | 'research';+ type: 'github' | 'research' | 'pdf';}export interface SearchRequest {@@ -294,7 +294,7 @@ export interface SearchRequest {sources?: Array<'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }>;- categories?: Array<'github' | 'research' | CategoryOption>;+ categories?: Array<'github' | 'research' | 'pdf' | CategoryOption>;limit?: number;tbs?: string;location?: string;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex ad9461a24..1e4965c52 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.3.7"+__version__ = "4.4.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 113cf64a2..244c07c82 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -123,7 +123,7 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:# Validate categories (if provided)if request.categories is not None:- valid_categories = {"github", "research"}+ valid_categories = {"github", "research", "pdf"}for category in request.categories:if isinstance(category, str):if category not in valid_categories:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 5b37a6776..6d41f3a41 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -186,7 +186,13 @@ class Source(BaseModel):SourceOption = Union[str, Source]class Category(BaseModel):- """Configuration for a search category."""+ """Configuration for a search category.++ Supported categories:+ - "github": Filter results to GitHub repositories+ - "research": Filter results to research papers and academic sites+ - "pdf": Filter results to PDF files (adds filetype:pdf to search)+ """type: strCategoryOption = Union[str, Category]