Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add the code search category
Karan Lokchandani2f5ccf3af312
diff --git a/apps/api/src/__tests__/snips/v2/key-restriction.test.ts b/apps/api/src/__tests__/snips/v2/key-restriction.test.tsindex fd68eb099..cc8050817 100644--- a/apps/api/src/__tests__/snips/v2/key-restriction.test.ts+++ b/apps/api/src/__tests__/snips/v2/key-restriction.test.ts@@ -1,5 +1,5 @@import { eq } from "drizzle-orm";-import { idmux, map, scrape, scrapeRaw, scrapeTimeout } from "./lib";+import { idmux, map, scrape, scrapeRaw, scrapeTimeout, searchRaw } from "./lib";import { createTestIdUrl, describeIf, Identity, TEST_PRODUCTION } from "../lib";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";@@ -137,6 +137,32 @@ describeIf(TEST_PRODUCTION)(scrapeTimeout * 2,);+ it.concurrent(+ "gates the code search category on the research group",+ async () => {+ const identity = await idmux({+ name: "key-restriction/code-category",+ credits: 10000,+ flags: { keyRestriction: true },+ });+ await seedRestriction(identity, { allowedEndpoints: ["search"] });++ const plain = await searchRaw({ query: "retries" }, identity);+ expect(plain.statusCode).toBe(200);++ const withCode = await searchRaw(+ { query: "retries", categories: ["code"] },+ identity,+ );+ expect(withCode.statusCode).toBe(403);+ expect(withCode.body.success).toBe(false);+ expect(withCode.body.error).toContain(+ "restricted to the following endpoints",+ );+ },+ scrapeTimeout * 2,+ );+it.concurrent("does not restrict a flagged team before it configures a restriction",async () => {diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex ca624f5a4..1626e9368 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -1025,6 +1025,27 @@ describe("V2 Types Validation", () => {]);});+ it("should accept the code category and reject code search params", () => {+ expect(+ searchRequestSchema.parse({ query: "test", categories: ["code"] })+ .categories,+ ).toEqual([{ type: "code" }]);++ expect(+ searchRequestSchema.parse({+ query: "test",+ categories: [{ type: "code" }],+ }).categories,+ ).toEqual([{ type: "code" }]);++ expect(() =>+ searchRequestSchema.parse({+ query: "test",+ categories: [{ type: "code", repos: ["firecrawl/firecrawl"] }],+ }),+ ).toThrow();+ });+it("should accept search request with advanced categories format", () => {const input: SearchRequestInput = {query: "test",diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 15e6d1a58..96c6bec53 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -1,9 +1,8 @@import express, { Request, Response } from "express";-import { Agent, fetch } from "undici";import { z } from "zod";import { v7 as uuidv7 } from "uuid";-import { config } from "../../config";import { logger as rootLogger } from "../../lib/logger";+import { fetchResearchUpstream } from "../../lib/research-upstream";import { chargeKeylessCredits } from "../../lib/keyless";import { billTeam } from "../../services/billing/credit_billing";import { getSearchForcedKind } from "../../lib/zdr-helpers";@@ -19,19 +18,12 @@ import type { RequestWithAuth } from "../v1/types";import { wrap } from "../../routes/shared";import { integrationSchema } from "../../utils/integration";-const TIMEOUT_MS = 120_000;const SEARCH_CREDITS_PER_TEN_RESULTS = 2;const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];-const dispatcher = new Agent({- connectTimeout: TIMEOUT_MS,- headersTimeout: TIMEOUT_MS,- bodyTimeout: TIMEOUT_MS,-});-const multiString = z.union([z.string(), z.array(z.string())]).optional()@@ -197,25 +189,6 @@ function addLegacySnakeCaseAliases<T>(value: T): T {return object as T;}-function appendQuery(- url: URL,- params: Record<string, unknown>,- allowed: string[],-) {- for (const key of allowed) {- const value = params[key];- if (Array.isArray(value)) {- for (const item of value) {- if (item !== undefined && item !== null) {- url.searchParams.append(key, String(item));- }- }- } else if (value !== undefined && value !== null) {- url.searchParams.append(key, String(value));- }- }-}-function resultCount(body: any): number {return Array.isArray(body?.results) ? body.results.length : 0;}@@ -257,18 +230,12 @@ function researchError(});}-async function fetchResearchUpstream(+async function fetchForRequest(req: RequestWithAuth<any, any, any>,path: string,params: Record<string, unknown>,queryKeys: string[],) {- const base = config.RESEARCH_PROXY_URL;- if (!base) return null;-- const url = new URL(base.replace(/\/+$/, "") + path);- appendQuery(url, params, queryKeys);-const headers: Record<string, string> = {};for (const h of FORWARDED_REQUEST_HEADERS) {const v = req.headers[h];@@ -276,12 +243,7 @@ async function fetchResearchUpstream(}headers["firecrawl-team-id"] = req.auth.team_id;- return fetch(url, {- method: "GET",- headers,- signal: AbortSignal.timeout(TIMEOUT_MS),- dispatcher,- });+ return fetchResearchUpstream({ path, params, queryKeys, headers });}function createResearchController(@@ -333,7 +295,7 @@ function createResearchController(let credits = 0;try {- const upstream = await fetchResearchUpstream(+ const upstream = await fetchForRequest(authedReq,endpoint.upstreamPath(params, authedReq),params,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 55dba5ad6..237722ba0 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -31,9 +31,11 @@ import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";import {actionTypesOf,+ checkKeyEndpointRestriction,checkKeyFormatRestriction,formatTypesOf,} from "../../lib/key-restriction";+import { wantsCodeCategory } from "../../search/code";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -85,6 +87,20 @@ export async function searchController(});}+ if (wantsCodeCategory(req.body.categories as CategoryOption[])) {+ const codeRestriction = await checkKeyEndpointRestriction(+ "/v2/code/search",+ req.acuc?.api_key_id,+ req.acuc?.flags ?? null,+ );+ if (!codeRestriction.allowed) {+ return res.status(codeRestriction.status).json({+ success: false,+ error: codeRestriction.error,+ });+ }+ }+if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex fe104e09d..952bc7f38 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1909,6 +1909,10 @@ const pdfCategoryOptions = z.strictObject({type: z.literal("pdf"),});+const codeCategoryOptions = z.strictObject({+ type: z.literal("code"),+});+const searchDomainSchema = z.string().trim()@@ -1945,13 +1949,14 @@ export const searchRequestSchema = zcategories: z.union([// Array of strings (simple format)- z.array(z.enum(["github", "research", "pdf"])),+ z.array(z.enum(["github", "research", "pdf", "code"])),// Array of objects (advanced format)z.array(z.union([githubCategoryOptions,researchCategoryOptions,pdfCategoryOptions,+ codeCategoryOptions,]),),])@@ -2086,6 +2091,10 @@ export const searchRequestSchema = zreturn {type: "pdf" as const,};+ case "code":+ return {+ type: "code" as const,+ };default:return { type: c as any };}diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 0d5dac8ee..e8ffc8926 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -167,6 +167,7 @@ export interface SearchV2Response {web?: WebSearchResult[];images?: ImageSearchResult[];news?: NewsSearchResult[];+ code?: WebSearchResult[];}export interface ScrapeActionContent {diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsnew file mode 100644index 000000000..b24673e23--- /dev/null+++ b/apps/api/src/lib/research-upstream.ts@@ -0,0 +1,50 @@+import { Agent, fetch } from "undici";+import { config } from "../config";++const TIMEOUT_MS = 120_000;++const dispatcher = new Agent({+ connectTimeout: TIMEOUT_MS,+ headersTimeout: TIMEOUT_MS,+ bodyTimeout: TIMEOUT_MS,+});++export function appendQuery(+ url: URL,+ params: Record<string, unknown>,+ allowed: string[],+) {+ for (const key of allowed) {+ const value = params[key];+ if (Array.isArray(value)) {+ for (const item of value) {+ if (item !== undefined && item !== null) {+ url.searchParams.append(key, String(item));+ }+ }+ } else if (value !== undefined && value !== null) {+ url.searchParams.append(key, String(value));+ }+ }+}++export async function fetchResearchUpstream(options: {+ path: string;+ params: Record<string, unknown>;+ queryKeys: string[];+ headers: Record<string, string>;+ timeoutMs?: number;+}) {+ const base = config.RESEARCH_PROXY_URL;+ if (!base) return null;++ const url = new URL(base.replace(/\/+$/, "") + options.path);+ appendQuery(url, options.params, options.queryKeys);++ return fetch(url, {+ method: "GET",+ headers: options.headers,+ signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),+ dispatcher,+ });+}diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex 8a25aa1a1..467a001cf 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -4,7 +4,7 @@*/interface CategoryInput {- type: "github" | "research" | "pdf";+ type: "github" | "research" | "pdf" | "code";sites?: string[];}diff --git a/apps/api/src/search/code.test.ts b/apps/api/src/search/code.test.tsnew file mode 100644index 000000000..b7fa46a33--- /dev/null+++ b/apps/api/src/search/code.test.ts@@ -0,0 +1,112 @@+vi.mock("undici", () => ({+ Agent: class {},+ fetch: vi.fn(),+}));++vi.mock("../config", () => ({+ config: { RESEARCH_PROXY_URL: "https://research.test/" },+}));++import { fetch } from "undici";+import { searchCodeCategory, wantsCodeCategory } from "./code";++const fetchMock = vi.mocked(fetch);++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+} as any;++function upstreamOk(body: unknown) {+ return { ok: true, status: 200, json: async () => body } as any;+}++function firstCall(): any[] {+ return fetchMock.mock.calls[0] as any[];+}++afterEach(() => {+ vi.clearAllMocks();+});++describe("wantsCodeCategory", () => {+ it("reads both the string form and the object form", () => {+ expect(wantsCodeCategory(["code"])).toBe(true);+ expect(wantsCodeCategory([{ type: "code" }])).toBe(true);+ expect(wantsCodeCategory(["github", "research"])).toBe(false);+ expect(wantsCodeCategory(undefined)).toBe(false);+ });+});++describe("searchCodeCategory", () => {+ it("sends the query and the result count, and nothing else", async () => {+ fetchMock.mockResolvedValue(upstreamOk({ results: [] }));++ await searchCodeCategory(+ { query: '"exact phrase" retries', limit: 7, teamId: "t1", timeout: 500 },+ logger,+ );++ const url = firstCall()[0] as URL;+ expect(url.pathname).toBe("/v2/code/search");+ expect(url.searchParams.get("query")).toBe('"exact phrase" retries');+ expect(url.searchParams.get("k")).toBe("7");+ expect([...url.searchParams.keys()]).toEqual(["query", "k"]);+ expect(firstCall()[1].method).toBe("GET");+ expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });+ });++ it("maps upstream results into web result shape", async () => {+ fetchMock.mockResolvedValue(+ upstreamOk({+ results: [+ {+ id: "abc",+ type: "issue",+ url: "https://github.com/a/b/issues/1",+ passages: [{ text: "first passage" }],+ },+ { id: "def", type: "readme", url: "", passages: [] },+ ],+ }),+ );++ const results = await searchCodeCategory(+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ logger,+ );++ expect(results).toEqual([+ {+ url: "https://github.com/a/b/issues/1",+ title: "https://github.com/a/b/issues/1",+ description: "first passage",+ position: 1,+ category: "code",+ },+ ]);+ });++ it("returns no results when the upstream fails", async () => {+ fetchMock.mockResolvedValue({ ok: false, status: 503 } as any);++ await expect(+ searchCodeCategory(+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ logger,+ ),+ ).resolves.toEqual([]);+ });++ it("returns no results when the upstream throws", async () => {+ fetchMock.mockRejectedValue(new Error("connection reset"));++ await expect(+ searchCodeCategory(+ { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ logger,+ ),+ ).resolves.toEqual([]);+ });+});diff --git a/apps/api/src/search/code.ts b/apps/api/src/search/code.tsnew file mode 100644index 000000000..6cdb58ea8--- /dev/null+++ b/apps/api/src/search/code.ts@@ -0,0 +1,57 @@+import type { Logger } from "winston";+import type { WebSearchResult } from "../lib/entities";+import { fetchResearchUpstream } from "../lib/research-upstream";+import type { CategoryOption } from "../lib/search-query-builder";++const CODE_QUERY_KEYS = ["query", "k"];++export function wantsCodeCategory(categories?: CategoryOption[]): boolean {+ return (categories ?? []).some(category =>+ typeof category === "string"+ ? category === "code"+ : category.type === "code",+ );+}++export async function searchCodeCategory(+ options: { query: string; limit: number; teamId: string; timeout: number },+ logger: Logger,+): Promise<WebSearchResult[]> {+ try {+ const upstream = await fetchResearchUpstream({+ path: "/v2/code/search",+ params: { query: options.query, k: options.limit },+ queryKeys: CODE_QUERY_KEYS,+ headers: { "firecrawl-team-id": options.teamId },+ timeoutMs: options.timeout,+ });+ if (upstream === null) {+ return [];+ }+ if (!upstream.ok) {+ logger.warn("Code category upstream failed", {+ status: upstream.status,+ });+ return [];+ }++ const body: any = await upstream.json();+ const results: any[] = Array.isArray(body?.results) ? body.results : [];+ return results+ .slice(0, options.limit)+ .map((result, index) => ({+ url: typeof result?.url === "string" ? result.url : "",+ title: typeof result?.url === "string" ? result.url : "",+ description:+ typeof result?.passages?.[0]?.text === "string"+ ? result.passages[0].text+ : "",+ position: index + 1,+ category: "code",+ }))+ .filter(result => result.url.length > 0);+ } catch (error) {+ logger.warn("Code category upstream error", { error });+ return [];+ }+}diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 15f5186ef..95297f2d0 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -13,6 +13,7 @@ import {mergeScrapedContent,calculateScrapeCredits,} from "./scrape";+import { searchCodeCategory, wantsCodeCategory } from "./code";import {highlightsEnvReady,runIndexedSearchHighlights,@@ -104,6 +105,13 @@ export async function executeSearch(},);+ const codeResults = wantsCodeCategory(categories)+ ? searchCodeCategory(+ { query, limit, teamId, timeout: options.timeout },+ logger,+ )+ : null;+const searchResponse = (await search({query: searchQuery,logger,@@ -197,6 +205,14 @@ export async function executeSearch(totalResultsCount += searchResponse.news.length;}+ if (codeResults) {+ const code = await codeResults;+ if (code.length > 0) {+ searchResponse.code = code;+ totalResultsCount += code.length;+ }+ }+const isZDR = options.enterprise?.includes("zdr");const creditsPerTenResults = isZDR ? 10 : 2;// Threat protection scan fees ride on the search credits: they are part of