Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): remove the github search category with a precise 400 (#5538)
Kris Oei0325cf38f7df
diff --git a/apps/api/src/__tests__/lib/search-query-builder.test.ts b/apps/api/src/__tests__/lib/search-query-builder.test.tsindex ae909f828..704e33952 100644--- a/apps/api/src/__tests__/lib/search-query-builder.test.ts+++ b/apps/api/src/__tests__/lib/search-query-builder.test.ts@@ -19,13 +19,6 @@ describe("Search Query Builder", () => {expect(result.categoryMap.size).toBe(0);});- it("should add GitHub filter for simple github category", () => {- const result = buildSearchQuery("web scraping", ["github"]);- expect(result.query).toBe("web scraping (site:github.com)");- expect(result.categoryMap.get("github.com")).toBe("github");- expect(result.categoryMap.size).toBe(1);- });-it("should add default research sites for simple research category", () => {const result = buildSearchQuery("neural networks", ["research"]);expect(result.query).toContain("neural networks (");@@ -41,21 +34,15 @@ describe("Search Query Builder", () => {});it("should handle multiple simple categories", () => {- const result = buildSearchQuery("firecrawl", ["github", "research"]);+ const result = buildSearchQuery("firecrawl", ["pdf", "research"]);expect(result.query).toContain("firecrawl (");- expect(result.query).toContain("site:github.com");+ expect(result.query).toContain("filetype:pdf");expect(result.query).toContain("site:arxiv.org");expect(result.query).toContain(" OR ");- expect(result.categoryMap.get("github.com")).toBe("github");+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");expect(result.categoryMap.get("arxiv.org")).toBe("research");- expect(result.categoryMap.size).toBe(15); // 1 GitHub + 14 research- });-- it("should handle GitHub category as object", () => {- const result = buildSearchQuery("code review", [{ type: "github" }]);- expect(result.query).toBe("code review (site:github.com)");- expect(result.categoryMap.get("github.com")).toBe("github");+ expect(result.categoryMap.size).toBe(15); // PDF marker + 14 research});it("should use custom research sites when provided", () => {@@ -75,28 +62,26 @@ describe("Search Query Builder", () => {it("should handle mixed string and object categories", () => {const result = buildSearchQuery("AI research", [- "github",+ "pdf",{type: "research",sites: ["arxiv.org"],},]);- expect(result.query).toBe(- "AI research (site:github.com OR site:arxiv.org)",- );- expect(result.categoryMap.get("github.com")).toBe("github");+ expect(result.query).toBe("AI research (site:arxiv.org) filetype:pdf");+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");expect(result.categoryMap.get("arxiv.org")).toBe("research");expect(result.categoryMap.size).toBe(2);});it("should handle special characters in base query", () => {- const result = buildSearchQuery("C++ programming", ["github"]);- expect(result.query).toBe("C++ programming (site:github.com)");+ const result = buildSearchQuery("C++ programming", ["pdf"]);+ expect(result.query).toBe("C++ programming filetype:pdf");});it("should handle quotes in base query", () => {- const result = buildSearchQuery('"exact phrase" search', ["github"]);- expect(result.query).toBe('"exact phrase" search (site:github.com)');+ const result = buildSearchQuery('"exact phrase" search', ["pdf"]);+ expect(result.query).toBe('"exact phrase" search filetype:pdf');});it("should ignore invalid category types", () => {@@ -152,13 +137,11 @@ describe("Search Query Builder", () => {});it("should combine categories with domain filters", () => {- const result = buildSearchQuery("web scraping", ["github"], {+ const result = buildSearchQuery("web scraping", ["pdf"], {includeDomains: ["firecrawl.dev"],});- expect(result.query).toBe(- "web scraping (site:github.com) site:firecrawl.dev",- );- expect(result.categoryMap.get("github.com")).toBe("github");+ expect(result.query).toBe("web scraping filetype:pdf site:firecrawl.dev");+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");});});@@ -274,14 +257,14 @@ describe("Search Query Builder", () => {it("should handle complex real-world query", () => {const result = buildSearchQuery('machine learning "neural networks" site:stackoverflow.com',- ["github", "research"],+ ["research"],);// Should append category filters to existing queryexpect(result.query).toContain('machine learning "neural networks" site:stackoverflow.com',);- expect(result.query).toContain("(site:github.com OR site:arxiv.org");+ expect(result.query).toContain("(site:arxiv.org OR ");// Check URLs can be categorizedconst githubUrl = "https://github.com/tensorflow/tensorflow";@@ -292,8 +275,8 @@ describe("Search Query Builder", () => {});it("should handle empty query with categories", () => {- const result = buildSearchQuery("", ["github"]);- expect(result.query).toBe(" (site:github.com)");+ const result = buildSearchQuery("", ["pdf"]);+ expect(result.query).toBe(" filetype:pdf");});it("should build correct query for multiple custom research sites", () => {diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex 98acc6235..543e06b12 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -479,7 +479,7 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("stays silent for every other category",async () => {const raw = await searchRaw(- { query: "firecrawl", categories: ["github"] },+ { query: "firecrawl", categories: ["pdf"] },identity,);expect(raw.statusCode).toBe(200);diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex af6948d33..66e9d3fba 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -1512,13 +1512,13 @@ describe("V2 Types Validation", () => {it("should accept search request with simple categories array", () => {const input: SearchRequestInput = {query: "test",- categories: ["github", "research"],+ categories: ["pdf", "research"],};const result = searchRequestSchema.parse(input);// Categories are transformed from string array to object arrayexpect(result.categories).toEqual([- { type: "github" },+ { type: "pdf" },{ type: "research" },]);});@@ -1625,7 +1625,7 @@ describe("V2 Types Validation", () => {expect(() =>searchRequestSchema.parse({query: "test",- categories: ["docs", "github", "developer_index"],+ categories: ["docs", "pdf", "developer_index"],}),).toThrow(/cannot be combined/);});@@ -1654,9 +1654,6 @@ describe("V2 Types Validation", () => {const input: SearchRequestInput = {query: "test",categories: [- {- type: "github",- },{type: "research",},diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex 4c39d630b..3b6741e65 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -216,7 +216,7 @@ describe("developer category code_searches ledger", () => {executeResult({ response: { web: [] } }),);- for (const categories of [undefined, ["github"]]) {+ for (const categories of [undefined, ["pdf"]]) {mockLogResearchEndpoint.mockClear();const req = makeReq({query: "http client",diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex d7adaa691..e8987d027 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -75,10 +75,13 @@ const IMAGE_SEARCH_US_ONLY_MESSAGE =const RESEARCH_CATEGORY_NOTICE: Notice = {message:- "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The github, pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",+ "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",links: ['<https://docs.firecrawl.dev/features/research>; rel="help"'],};+const GITHUB_CATEGORY_REMOVED =+ "The 'github' search category was removed. To search GitHub issues, pull requests and READMEs, use the 'developer' category or GET or POST /v2/search/developer (https://docs.firecrawl.dev/features/developer). To keep web results from github.com, use includeDomains: [\"github.com\"].";+// Ahead of auth and validation so rejected requests carry the notice too.export function researchCategoryNoticeMiddleware(req: Request,@@ -165,6 +168,17 @@ async function searchControllerInner(typeof req.body?.origin === "string" ? req.body.origin : undefined;delete (req.body as Record<string, unknown>)?.objective;delete (req.body as Record<string, unknown>)?.clientModel;+ if (hasCategory(req.body?.categories, "github")) {+ logger.warn("Removed search category requested", {+ module: "deprecations",+ category: "github",+ origin: requestOrigin({ origin: rawOrigin }, req),+ });+ return res.status(400).json({+ success: false,+ error: GITHUB_CATEGORY_REMOVED,+ });+ }const sourcesOmitted =(req.body as Record<string, unknown>)?.sources === undefined;req.body = searchRequestSchema.parse(req.body);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 51e672d0d..f65ce2c75 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2519,10 +2519,6 @@ const newsSearchSourceOptions = z.strictObject({});// Category source type definitions-const githubCategoryOptions = z.strictObject({- type: z.literal("github"),-});-const researchCategoryOptions = z.strictObject({type: z.literal("research"),});@@ -2621,10 +2617,9 @@ export const searchRequestSchema = z.preprocess(normalizeDeveloperCategoryAliases,z.union([- z.array(z.enum(["github", "research", "pdf", "developer", "gov"])),+ z.array(z.enum(["research", "pdf", "developer", "gov"])),z.array(z.union([- githubCategoryOptions,researchCategoryOptions,pdfCategoryOptions,developerCategoryOptions,@@ -2779,10 +2774,6 @@ export const searchRequestSchema = z// It's a string array, transform to object arraycategories = (categories as string[]).map(c => {switch (c) {- case "github":- return {- type: "github" as const,- };case "research":return {type: "research" as const,diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex 7b3e531e1..71c5cfca4 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -4,7 +4,7 @@*/interface CategoryInput {- type: "github" | "research" | "pdf" | "developer" | "gov";+ type: "research" | "pdf" | "developer" | "gov";sites?: string[];}@@ -40,7 +40,7 @@ const DEFAULT_RESEARCH_SITES = [export function hasCategory(categories: unknown,- type: CategoryInput["type"],+ type: CategoryInput["type"] | "github",): boolean {return (Array.isArray(categories) &&@@ -67,10 +67,7 @@ export function buildSearchQuery(for (const category of categories ?? []) {if (typeof category === "string") {// Simple string format- if (category === "github") {- siteFilters.push("site:github.com");- categoryMap.set("github.com", "github");- } else if (category === "research") {+ if (category === "research") {// Use default research sitesfor (const site of DEFAULT_RESEARCH_SITES) {siteFilters.push(`site:${site}`);@@ -82,10 +79,7 @@ export function buildSearchQuery(}} else {// Object format with options- if (category.type === "github") {- siteFilters.push("site:github.com");- categoryMap.set("github.com", "github");- } else if (category.type === "research") {+ if (category.type === "research") {// Use custom sites if provided, otherwise defaultsconst sites = category.sites || DEFAULT_RESEARCH_SITES;for (const site of sites) {diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex dc1f531e2..26aaead72 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -591,7 +591,7 @@ describe("executeSearch developer category", () => {it("rejects developer combined with other categories at the schema", () => {const mixed = searchRequestSchema.safeParse({query: "retries",- categories: ["developer", "github"],+ categories: ["developer", "pdf"],});expect(mixed.success).toBe(false);if (!mixed.success) {