Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add the code search category

Karan Lokchandani2f5ccf3af312
diff --git a/apps/api/src/__tests__/snips/v2/key-restriction.test.ts b/apps/api/src/__tests__/snips/v2/key-restriction.test.ts
index fd68eb099..cc8050817 100644
--- a/apps/api/src/__tests__/snips/v2/key-restriction.test.ts
+++ b/apps/api/src/__tests__/snips/v2/key-restriction.test.ts
@@ -1,5 +1,5 @@
import { eq } from "drizzle-orm";
-import { idmux, map, scrape, scrapeRaw, scrapeTimeout } from "./lib";
+import { idmux, map, scrape, scrapeRaw, scrapeTimeout, searchRaw } from "./lib";
import { createTestIdUrl, describeIf, Identity, TEST_PRODUCTION } from "../lib";
import { db } from "../../../db/connection";
import * as schema from "../../../db/schema";
@@ -137,6 +137,32 @@ describeIf(TEST_PRODUCTION)(
scrapeTimeout * 2,
);
+ it.concurrent(
+ "gates the code search category on the research group",
+ async () => {
+ const identity = await idmux({
+ name: "key-restriction/code-category",
+ credits: 10000,
+ flags: { keyRestriction: true },
+ });
+ await seedRestriction(identity, { allowedEndpoints: ["search"] });
+
+ const plain = await searchRaw({ query: "retries" }, identity);
+ expect(plain.statusCode).toBe(200);
+
+ const withCode = await searchRaw(
+ { query: "retries", categories: ["code"] },
+ identity,
+ );
+ expect(withCode.statusCode).toBe(403);
+ expect(withCode.body.success).toBe(false);
+ expect(withCode.body.error).toContain(
+ "restricted to the following endpoints",
+ );
+ },
+ scrapeTimeout * 2,
+ );
+
it.concurrent(
"does not restrict a flagged team before it configures a restriction",
async () => {
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index ca624f5a4..1626e9368 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -1025,6 +1025,27 @@ describe("V2 Types Validation", () => {
]);
});
+ it("should accept the code category and reject code search params", () => {
+ expect(
+ searchRequestSchema.parse({ query: "test", categories: ["code"] })
+ .categories,
+ ).toEqual([{ type: "code" }]);
+
+ expect(
+ searchRequestSchema.parse({
+ query: "test",
+ categories: [{ type: "code" }],
+ }).categories,
+ ).toEqual([{ type: "code" }]);
+
+ expect(() =>
+ searchRequestSchema.parse({
+ query: "test",
+ categories: [{ type: "code", repos: ["firecrawl/firecrawl"] }],
+ }),
+ ).toThrow();
+ });
+
it("should accept search request with advanced categories format", () => {
const input: SearchRequestInput = {
query: "test",
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 15e6d1a58..96c6bec53 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -1,9 +1,8 @@
import express, { Request, Response } from "express";
-import { Agent, fetch } from "undici";
import { z } from "zod";
import { v7 as uuidv7 } from "uuid";
-import { config } from "../../config";
import { logger as rootLogger } from "../../lib/logger";
+import { fetchResearchUpstream } from "../../lib/research-upstream";
import { chargeKeylessCredits } from "../../lib/keyless";
import { billTeam } from "../../services/billing/credit_billing";
import { getSearchForcedKind } from "../../lib/zdr-helpers";
@@ -19,19 +18,12 @@ import type { RequestWithAuth } from "../v1/types";
import { wrap } from "../../routes/shared";
import { integrationSchema } from "../../utils/integration";
-const TIMEOUT_MS = 120_000;
const SEARCH_CREDITS_PER_TEN_RESULTS = 2;
const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;
const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];
const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];
-const dispatcher = new Agent({
- connectTimeout: TIMEOUT_MS,
- headersTimeout: TIMEOUT_MS,
- bodyTimeout: TIMEOUT_MS,
-});
-
const multiString = z
.union([z.string(), z.array(z.string())])
.optional()
@@ -197,25 +189,6 @@ function addLegacySnakeCaseAliases<T>(value: T): T {
return object as T;
}
-function appendQuery(
- url: URL,
- params: Record<string, unknown>,
- allowed: string[],
-) {
- for (const key of allowed) {
- const value = params[key];
- if (Array.isArray(value)) {
- for (const item of value) {
- if (item !== undefined && item !== null) {
- url.searchParams.append(key, String(item));
- }
- }
- } else if (value !== undefined && value !== null) {
- url.searchParams.append(key, String(value));
- }
- }
-}
-
function resultCount(body: any): number {
return Array.isArray(body?.results) ? body.results.length : 0;
}
@@ -257,18 +230,12 @@ function researchError(
});
}
-async function fetchResearchUpstream(
+async function fetchForRequest(
req: RequestWithAuth<any, any, any>,
path: string,
params: Record<string, unknown>,
queryKeys: string[],
) {
- const base = config.RESEARCH_PROXY_URL;
- if (!base) return null;
-
- const url = new URL(base.replace(/\/+$/, "") + path);
- appendQuery(url, params, queryKeys);
-
const headers: Record<string, string> = {};
for (const h of FORWARDED_REQUEST_HEADERS) {
const v = req.headers[h];
@@ -276,12 +243,7 @@ async function fetchResearchUpstream(
}
headers["firecrawl-team-id"] = req.auth.team_id;
- return fetch(url, {
- method: "GET",
- headers,
- signal: AbortSignal.timeout(TIMEOUT_MS),
- dispatcher,
- });
+ return fetchResearchUpstream({ path, params, queryKeys, headers });
}
function createResearchController(
@@ -333,7 +295,7 @@ function createResearchController(
let credits = 0;
try {
- const upstream = await fetchResearchUpstream(
+ const upstream = await fetchForRequest(
authedReq,
endpoint.upstreamPath(params, authedReq),
params,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 55dba5ad6..237722ba0 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -31,9 +31,11 @@ import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import {
actionTypesOf,
+ checkKeyEndpointRestriction,
checkKeyFormatRestriction,
formatTypesOf,
} from "../../lib/key-restriction";
+import { wantsCodeCategory } from "../../search/code";
export async function searchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
@@ -85,6 +87,20 @@ export async function searchController(
});
}
+ if (wantsCodeCategory(req.body.categories as CategoryOption[])) {
+ const codeRestriction = await checkKeyEndpointRestriction(
+ "/v2/code/search",
+ req.acuc?.api_key_id,
+ req.acuc?.flags ?? null,
+ );
+ if (!codeRestriction.allowed) {
+ return res.status(codeRestriction.status).json({
+ success: false,
+ error: codeRestriction.error,
+ });
+ }
+ }
+
if (
req.body.__agentInterop &&
config.AGENT_INTEROP_SECRET &&
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index fe104e09d..952bc7f38 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1909,6 +1909,10 @@ const pdfCategoryOptions = z.strictObject({
type: z.literal("pdf"),
});
+const codeCategoryOptions = z.strictObject({
+ type: z.literal("code"),
+});
+
const searchDomainSchema = z
.string()
.trim()
@@ -1945,13 +1949,14 @@ export const searchRequestSchema = z
categories: z
.union([
// Array of strings (simple format)
- z.array(z.enum(["github", "research", "pdf"])),
+ z.array(z.enum(["github", "research", "pdf", "code"])),
// Array of objects (advanced format)
z.array(
z.union([
githubCategoryOptions,
researchCategoryOptions,
pdfCategoryOptions,
+ codeCategoryOptions,
]),
),
])
@@ -2086,6 +2091,10 @@ export const searchRequestSchema = z
return {
type: "pdf" as const,
};
+ case "code":
+ return {
+ type: "code" as const,
+ };
default:
return { type: c as any };
}
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 0d5dac8ee..e8ffc8926 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -167,6 +167,7 @@ export interface SearchV2Response {
web?: WebSearchResult[];
images?: ImageSearchResult[];
news?: NewsSearchResult[];
+ code?: WebSearchResult[];
}
export interface ScrapeActionContent {
diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.ts
new file mode 100644
index 000000000..b24673e23
--- /dev/null
+++ b/apps/api/src/lib/research-upstream.ts
@@ -0,0 +1,50 @@
+import { Agent, fetch } from "undici";
+import { config } from "../config";
+
+const TIMEOUT_MS = 120_000;
+
+const dispatcher = new Agent({
+ connectTimeout: TIMEOUT_MS,
+ headersTimeout: TIMEOUT_MS,
+ bodyTimeout: TIMEOUT_MS,
+});
+
+export function appendQuery(
+ url: URL,
+ params: Record<string, unknown>,
+ allowed: string[],
+) {
+ for (const key of allowed) {
+ const value = params[key];
+ if (Array.isArray(value)) {
+ for (const item of value) {
+ if (item !== undefined && item !== null) {
+ url.searchParams.append(key, String(item));
+ }
+ }
+ } else if (value !== undefined && value !== null) {
+ url.searchParams.append(key, String(value));
+ }
+ }
+}
+
+export async function fetchResearchUpstream(options: {
+ path: string;
+ params: Record<string, unknown>;
+ queryKeys: string[];
+ headers: Record<string, string>;
+ timeoutMs?: number;
+}) {
+ const base = config.RESEARCH_PROXY_URL;
+ if (!base) return null;
+
+ const url = new URL(base.replace(/\/+$/, "") + options.path);
+ appendQuery(url, options.params, options.queryKeys);
+
+ return fetch(url, {
+ method: "GET",
+ headers: options.headers,
+ signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
+ dispatcher,
+ });
+}
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index 8a25aa1a1..467a001cf 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -4,7 +4,7 @@
*/
interface CategoryInput {
- type: "github" | "research" | "pdf";
+ type: "github" | "research" | "pdf" | "code";
sites?: string[];
}
diff --git a/apps/api/src/search/code.test.ts b/apps/api/src/search/code.test.ts
new file mode 100644
index 000000000..b7fa46a33
--- /dev/null
+++ b/apps/api/src/search/code.test.ts
@@ -0,0 +1,112 @@
+vi.mock("undici", () => ({
+ Agent: class {},
+ fetch: vi.fn(),
+}));
+
+vi.mock("../config", () => ({
+ config: { RESEARCH_PROXY_URL: "https://research.test/" },
+}));
+
+import { fetch } from "undici";
+import { searchCodeCategory, wantsCodeCategory } from "./code";
+
+const fetchMock = vi.mocked(fetch);
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+} as any;
+
+function upstreamOk(body: unknown) {
+ return { ok: true, status: 200, json: async () => body } as any;
+}
+
+function firstCall(): any[] {
+ return fetchMock.mock.calls[0] as any[];
+}
+
+afterEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("wantsCodeCategory", () => {
+ it("reads both the string form and the object form", () => {
+ expect(wantsCodeCategory(["code"])).toBe(true);
+ expect(wantsCodeCategory([{ type: "code" }])).toBe(true);
+ expect(wantsCodeCategory(["github", "research"])).toBe(false);
+ expect(wantsCodeCategory(undefined)).toBe(false);
+ });
+});
+
+describe("searchCodeCategory", () => {
+ it("sends the query and the result count, and nothing else", async () => {
+ fetchMock.mockResolvedValue(upstreamOk({ results: [] }));
+
+ await searchCodeCategory(
+ { query: '"exact phrase" retries', limit: 7, teamId: "t1", timeout: 500 },
+ logger,
+ );
+
+ const url = firstCall()[0] as URL;
+ expect(url.pathname).toBe("/v2/code/search");
+ expect(url.searchParams.get("query")).toBe('"exact phrase" retries');
+ expect(url.searchParams.get("k")).toBe("7");
+ expect([...url.searchParams.keys()]).toEqual(["query", "k"]);
+ expect(firstCall()[1].method).toBe("GET");
+ expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });
+ });
+
+ it("maps upstream results into web result shape", async () => {
+ fetchMock.mockResolvedValue(
+ upstreamOk({
+ results: [
+ {
+ id: "abc",
+ type: "issue",
+ url: "https://github.com/a/b/issues/1",
+ passages: [{ text: "first passage" }],
+ },
+ { id: "def", type: "readme", url: "", passages: [] },
+ ],
+ }),
+ );
+
+ const results = await searchCodeCategory(
+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ logger,
+ );
+
+ expect(results).toEqual([
+ {
+ url: "https://github.com/a/b/issues/1",
+ title: "https://github.com/a/b/issues/1",
+ description: "first passage",
+ position: 1,
+ category: "code",
+ },
+ ]);
+ });
+
+ it("returns no results when the upstream fails", async () => {
+ fetchMock.mockResolvedValue({ ok: false, status: 503 } as any);
+
+ await expect(
+ searchCodeCategory(
+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ logger,
+ ),
+ ).resolves.toEqual([]);
+ });
+
+ it("returns no results when the upstream throws", async () => {
+ fetchMock.mockRejectedValue(new Error("connection reset"));
+
+ await expect(
+ searchCodeCategory(
+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ logger,
+ ),
+ ).resolves.toEqual([]);
+ });
+});
diff --git a/apps/api/src/search/code.ts b/apps/api/src/search/code.ts
new file mode 100644
index 000000000..6cdb58ea8
--- /dev/null
+++ b/apps/api/src/search/code.ts
@@ -0,0 +1,57 @@
+import type { Logger } from "winston";
+import type { WebSearchResult } from "../lib/entities";
+import { fetchResearchUpstream } from "../lib/research-upstream";
+import type { CategoryOption } from "../lib/search-query-builder";
+
+const CODE_QUERY_KEYS = ["query", "k"];
+
+export function wantsCodeCategory(categories?: CategoryOption[]): boolean {
+ return (categories ?? []).some(category =>
+ typeof category === "string"
+ ? category === "code"
+ : category.type === "code",
+ );
+}
+
+export async function searchCodeCategory(
+ options: { query: string; limit: number; teamId: string; timeout: number },
+ logger: Logger,
+): Promise<WebSearchResult[]> {
+ try {
+ const upstream = await fetchResearchUpstream({
+ path: "/v2/code/search",
+ params: { query: options.query, k: options.limit },
+ queryKeys: CODE_QUERY_KEYS,
+ headers: { "firecrawl-team-id": options.teamId },
+ timeoutMs: options.timeout,
+ });
+ if (upstream === null) {
+ return [];
+ }
+ if (!upstream.ok) {
+ logger.warn("Code category upstream failed", {
+ status: upstream.status,
+ });
+ return [];
+ }
+
+ const body: any = await upstream.json();
+ const results: any[] = Array.isArray(body?.results) ? body.results : [];
+ return results
+ .slice(0, options.limit)
+ .map((result, index) => ({
+ url: typeof result?.url === "string" ? result.url : "",
+ title: typeof result?.url === "string" ? result.url : "",
+ description:
+ typeof result?.passages?.[0]?.text === "string"
+ ? result.passages[0].text
+ : "",
+ position: index + 1,
+ category: "code",
+ }))
+ .filter(result => result.url.length > 0);
+ } catch (error) {
+ logger.warn("Code category upstream error", { error });
+ return [];
+ }
+}
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 15f5186ef..95297f2d0 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -13,6 +13,7 @@ import {
mergeScrapedContent,
calculateScrapeCredits,
} from "./scrape";
+import { searchCodeCategory, wantsCodeCategory } from "./code";
import {
highlightsEnvReady,
runIndexedSearchHighlights,
@@ -104,6 +105,13 @@ export async function executeSearch(
},
);
+ const codeResults = wantsCodeCategory(categories)
+ ? searchCodeCategory(
+ { query, limit, teamId, timeout: options.timeout },
+ logger,
+ )
+ : null;
+
const searchResponse = (await search({
query: searchQuery,
logger,
@@ -197,6 +205,14 @@ export async function executeSearch(
totalResultsCount += searchResponse.news.length;
}
+ if (codeResults) {
+ const code = await codeResults;
+ if (code.length > 0) {
+ searchResponse.code = code;
+ totalResultsCount += code.length;
+ }
+ }
+
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
// Threat protection scan fees ride on the search credits: they are part of