Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): remove the github search category with a precise 400 (#5538)

Kris Oei0325cf38f7df
diff --git a/apps/api/src/__tests__/lib/search-query-builder.test.ts b/apps/api/src/__tests__/lib/search-query-builder.test.ts
index ae909f828..704e33952 100644
--- a/apps/api/src/__tests__/lib/search-query-builder.test.ts
+++ b/apps/api/src/__tests__/lib/search-query-builder.test.ts
@@ -19,13 +19,6 @@ describe("Search Query Builder", () => {
expect(result.categoryMap.size).toBe(0);
});
- it("should add GitHub filter for simple github category", () => {
- const result = buildSearchQuery("web scraping", ["github"]);
- expect(result.query).toBe("web scraping (site:github.com)");
- expect(result.categoryMap.get("github.com")).toBe("github");
- expect(result.categoryMap.size).toBe(1);
- });
-
it("should add default research sites for simple research category", () => {
const result = buildSearchQuery("neural networks", ["research"]);
expect(result.query).toContain("neural networks (");
@@ -41,21 +34,15 @@ describe("Search Query Builder", () => {
});
it("should handle multiple simple categories", () => {
- const result = buildSearchQuery("firecrawl", ["github", "research"]);
+ const result = buildSearchQuery("firecrawl", ["pdf", "research"]);
expect(result.query).toContain("firecrawl (");
- expect(result.query).toContain("site:github.com");
+ expect(result.query).toContain("filetype:pdf");
expect(result.query).toContain("site:arxiv.org");
expect(result.query).toContain(" OR ");
- expect(result.categoryMap.get("github.com")).toBe("github");
+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");
expect(result.categoryMap.get("arxiv.org")).toBe("research");
- expect(result.categoryMap.size).toBe(15); // 1 GitHub + 14 research
- });
-
- it("should handle GitHub category as object", () => {
- const result = buildSearchQuery("code review", [{ type: "github" }]);
- expect(result.query).toBe("code review (site:github.com)");
- expect(result.categoryMap.get("github.com")).toBe("github");
+ expect(result.categoryMap.size).toBe(15); // PDF marker + 14 research
});
it("should use custom research sites when provided", () => {
@@ -75,28 +62,26 @@ describe("Search Query Builder", () => {
it("should handle mixed string and object categories", () => {
const result = buildSearchQuery("AI research", [
- "github",
+ "pdf",
{
type: "research",
sites: ["arxiv.org"],
},
]);
- expect(result.query).toBe(
- "AI research (site:github.com OR site:arxiv.org)",
- );
- expect(result.categoryMap.get("github.com")).toBe("github");
+ expect(result.query).toBe("AI research (site:arxiv.org) filetype:pdf");
+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");
expect(result.categoryMap.get("arxiv.org")).toBe("research");
expect(result.categoryMap.size).toBe(2);
});
it("should handle special characters in base query", () => {
- const result = buildSearchQuery("C++ programming", ["github"]);
- expect(result.query).toBe("C++ programming (site:github.com)");
+ const result = buildSearchQuery("C++ programming", ["pdf"]);
+ expect(result.query).toBe("C++ programming filetype:pdf");
});
it("should handle quotes in base query", () => {
- const result = buildSearchQuery('"exact phrase" search', ["github"]);
- expect(result.query).toBe('"exact phrase" search (site:github.com)');
+ const result = buildSearchQuery('"exact phrase" search', ["pdf"]);
+ expect(result.query).toBe('"exact phrase" search filetype:pdf');
});
it("should ignore invalid category types", () => {
@@ -152,13 +137,11 @@ describe("Search Query Builder", () => {
});
it("should combine categories with domain filters", () => {
- const result = buildSearchQuery("web scraping", ["github"], {
+ const result = buildSearchQuery("web scraping", ["pdf"], {
includeDomains: ["firecrawl.dev"],
});
- expect(result.query).toBe(
- "web scraping (site:github.com) site:firecrawl.dev",
- );
- expect(result.categoryMap.get("github.com")).toBe("github");
+ expect(result.query).toBe("web scraping filetype:pdf site:firecrawl.dev");
+ expect(result.categoryMap.get("__pdf__")).toBe("pdf");
});
});
@@ -274,14 +257,14 @@ describe("Search Query Builder", () => {
it("should handle complex real-world query", () => {
const result = buildSearchQuery(
'machine learning "neural networks" site:stackoverflow.com',
- ["github", "research"],
+ ["research"],
);
// Should append category filters to existing query
expect(result.query).toContain(
'machine learning "neural networks" site:stackoverflow.com',
);
- expect(result.query).toContain("(site:github.com OR site:arxiv.org");
+ expect(result.query).toContain("(site:arxiv.org OR ");
// Check URLs can be categorized
const githubUrl = "https://github.com/tensorflow/tensorflow";
@@ -292,8 +275,8 @@ describe("Search Query Builder", () => {
});
it("should handle empty query with categories", () => {
- const result = buildSearchQuery("", ["github"]);
- expect(result.query).toBe(" (site:github.com)");
+ const result = buildSearchQuery("", ["pdf"]);
+ expect(result.query).toBe(" filetype:pdf");
});
it("should build correct query for multiple custom research sites", () => {
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index 98acc6235..543e06b12 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -479,7 +479,7 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)(
"stays silent for every other category",
async () => {
const raw = await searchRaw(
- { query: "firecrawl", categories: ["github"] },
+ { query: "firecrawl", categories: ["pdf"] },
identity,
);
expect(raw.statusCode).toBe(200);
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index af6948d33..66e9d3fba 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -1512,13 +1512,13 @@ describe("V2 Types Validation", () => {
it("should accept search request with simple categories array", () => {
const input: SearchRequestInput = {
query: "test",
- categories: ["github", "research"],
+ categories: ["pdf", "research"],
};
const result = searchRequestSchema.parse(input);
// Categories are transformed from string array to object array
expect(result.categories).toEqual([
- { type: "github" },
+ { type: "pdf" },
{ type: "research" },
]);
});
@@ -1625,7 +1625,7 @@ describe("V2 Types Validation", () => {
expect(() =>
searchRequestSchema.parse({
query: "test",
- categories: ["docs", "github", "developer_index"],
+ categories: ["docs", "pdf", "developer_index"],
}),
).toThrow(/cannot be combined/);
});
@@ -1654,9 +1654,6 @@ describe("V2 Types Validation", () => {
const input: SearchRequestInput = {
query: "test",
categories: [
- {
- type: "github",
- },
{
type: "research",
},
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
index 4c39d630b..3b6741e65 100644
--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
@@ -216,7 +216,7 @@ describe("developer category code_searches ledger", () => {
executeResult({ response: { web: [] } }),
);
- for (const categories of [undefined, ["github"]]) {
+ for (const categories of [undefined, ["pdf"]]) {
mockLogResearchEndpoint.mockClear();
const req = makeReq({
query: "http client",
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index d7adaa691..e8987d027 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -75,10 +75,13 @@ const IMAGE_SEARCH_US_ONLY_MESSAGE =
const RESEARCH_CATEGORY_NOTICE: Notice = {
message:
- "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The github, pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",
+ "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",
links: ['<https://docs.firecrawl.dev/features/research>; rel="help"'],
};
+const GITHUB_CATEGORY_REMOVED =
+ "The 'github' search category was removed. To search GitHub issues, pull requests and READMEs, use the 'developer' category or GET or POST /v2/search/developer (https://docs.firecrawl.dev/features/developer). To keep web results from github.com, use includeDomains: [\"github.com\"].";
+
// Ahead of auth and validation so rejected requests carry the notice too.
export function researchCategoryNoticeMiddleware(
req: Request,
@@ -165,6 +168,17 @@ async function searchControllerInner(
typeof req.body?.origin === "string" ? req.body.origin : undefined;
delete (req.body as Record<string, unknown>)?.objective;
delete (req.body as Record<string, unknown>)?.clientModel;
+ if (hasCategory(req.body?.categories, "github")) {
+ logger.warn("Removed search category requested", {
+ module: "deprecations",
+ category: "github",
+ origin: requestOrigin({ origin: rawOrigin }, req),
+ });
+ return res.status(400).json({
+ success: false,
+ error: GITHUB_CATEGORY_REMOVED,
+ });
+ }
const sourcesOmitted =
(req.body as Record<string, unknown>)?.sources === undefined;
req.body = searchRequestSchema.parse(req.body);
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 51e672d0d..f65ce2c75 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2519,10 +2519,6 @@ const newsSearchSourceOptions = z.strictObject({
});
// Category source type definitions
-const githubCategoryOptions = z.strictObject({
- type: z.literal("github"),
-});
-
const researchCategoryOptions = z.strictObject({
type: z.literal("research"),
});
@@ -2621,10 +2617,9 @@ export const searchRequestSchema = z
.preprocess(
normalizeDeveloperCategoryAliases,
z.union([
- z.array(z.enum(["github", "research", "pdf", "developer", "gov"])),
+ z.array(z.enum(["research", "pdf", "developer", "gov"])),
z.array(
z.union([
- githubCategoryOptions,
researchCategoryOptions,
pdfCategoryOptions,
developerCategoryOptions,
@@ -2779,10 +2774,6 @@ export const searchRequestSchema = z
// It's a string array, transform to object array
categories = (categories as string[]).map(c => {
switch (c) {
- case "github":
- return {
- type: "github" as const,
- };
case "research":
return {
type: "research" as const,
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index 7b3e531e1..71c5cfca4 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -4,7 +4,7 @@
*/
interface CategoryInput {
- type: "github" | "research" | "pdf" | "developer" | "gov";
+ type: "research" | "pdf" | "developer" | "gov";
sites?: string[];
}
@@ -40,7 +40,7 @@ const DEFAULT_RESEARCH_SITES = [
export function hasCategory(
categories: unknown,
- type: CategoryInput["type"],
+ type: CategoryInput["type"] | "github",
): boolean {
return (
Array.isArray(categories) &&
@@ -67,10 +67,7 @@ export function buildSearchQuery(
for (const category of categories ?? []) {
if (typeof category === "string") {
// Simple string format
- if (category === "github") {
- siteFilters.push("site:github.com");
- categoryMap.set("github.com", "github");
- } else if (category === "research") {
+ if (category === "research") {
// Use default research sites
for (const site of DEFAULT_RESEARCH_SITES) {
siteFilters.push(`site:${site}`);
@@ -82,10 +79,7 @@ export function buildSearchQuery(
}
} else {
// Object format with options
- if (category.type === "github") {
- siteFilters.push("site:github.com");
- categoryMap.set("github.com", "github");
- } else if (category.type === "research") {
+ if (category.type === "research") {
// Use custom sites if provided, otherwise defaults
const sites = category.sites || DEFAULT_RESEARCH_SITES;
for (const site of sites) {
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index dc1f531e2..26aaead72 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -591,7 +591,7 @@ describe("executeSearch developer category", () => {
it("rejects developer combined with other categories at the schema", () => {
const mixed = searchRequestSchema.safeParse({
query: "retries",
- categories: ["developer", "github"],
+ categories: ["developer", "pdf"],
});
expect(mixed.success).toBe(false);
if (!mixed.success) {