Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
vi.mock("undici", () => ({
Agent: class {},
fetch: vi.fn(),
}));
vi.mock("../config", () => ({
config: { RESEARCH_PROXY_URL: "https://research.test/" },
}));
import { fetch } from "undici";
import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";
const fetchMock = vi.mocked(fetch);
const logger = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
} as any;
function upstreamOk(body: unknown) {
return { ok: true, status: 200, json: async () => body } as any;
}
function firstCall(): any[] {
return fetchMock.mock.calls[0] as any[];
}
afterEach(() => {
vi.clearAllMocks();
});
describe("wantsDeveloperCategory", () => {
it("reads both the string form and the object form", () => {
expect(wantsDeveloperCategory(["developer"])).toBe(true);
expect(wantsDeveloperCategory([{ type: "developer" }])).toBe(true);
expect(wantsDeveloperCategory(["github", "research"])).toBe(false);
expect(wantsDeveloperCategory(undefined)).toBe(false);
});
});
describe("searchDeveloperCategory", () => {
it("sends the query and the result count, and nothing else", async () => {
fetchMock.mockResolvedValue(upstreamOk({ results: [] }));
await searchDeveloperCategory(
{
query: '"exact phrase" retries',
limit: 7,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
},
logger,
);
const url = firstCall()[0] as URL;
expect(url.pathname).toBe("/v2/code/search");
expect(url.searchParams.get("query")).toBe('"exact phrase" retries');
expect(url.searchParams.get("k")).toBe("7");
expect([...url.searchParams.keys()]).toEqual(["query", "k"]);
expect(url.searchParams.has("skills")).toBe(false);
expect(firstCall()[1].method).toBe("GET");
expect(firstCall()[1].headers).toEqual({
"firecrawl-team-id": "t1",
"firecrawl-zero-data-retention": "false",
});
});
it("tells the upstream that a zero data retention search must not be kept", async () => {
fetchMock.mockResolvedValue(upstreamOk({ results: [] }));
await searchDeveloperCategory(
{
query: "private query",
limit: 3,
teamId: "t1",
timeout: 500,
zeroDataRetention: true,
},
logger,
);
expect(firstCall()[1].headers["firecrawl-zero-data-retention"]).toBe(
"true",
);
});
it("maps results to the exact web schema and leaks nothing upstream", async () => {
const license = { state: "licensed", spdx_id: "MIT" };
fetchMock.mockResolvedValue(
upstreamOk({
results: [
{
id: "issue:a/b#1",
url: "https://github.com/a/b/issues/1",
title: "Retries drop the backoff",
passages: [
{ text: "first passage", citation_url: "https://example.com" },
],
license,
},
{ id: "readme:a/b", url: "", passages: [] },
],
}),
);
const results = await searchDeveloperCategory(
{
query: "retries",
limit: 5,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
},
logger,
);
// Exact WebSearchResult shape: the upstream id/passages/license must not
// appear on the wire; the first passage becomes the description.
expect(results).toStrictEqual([
{
url: "https://github.com/a/b/issues/1",
title: "Retries drop the backoff",
description: "first passage",
position: 1,
category: "developer",
},
]);
});
it("falls back to the url when the upstream sends no usable title", async () => {
fetchMock.mockResolvedValue(
upstreamOk({
results: [
{
id: "abc",
type: "pull_request",
url: "https://github.com/a/b/pull/1",
passages: [{ text: "first passage" }],
},
{
id: "def",
type: "pull_request",
url: "https://github.com/a/b/pull/2",
title: " ",
passages: [],
},
],
}),
);
const results = await searchDeveloperCategory(
{
query: "retries",
limit: 5,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
},
logger,
);
expect(results.map(result => result.title)).toEqual([
"https://github.com/a/b/pull/1",
"https://github.com/a/b/pull/2",
]);
});
it("returns no results and drains the body when the upstream fails", async () => {
const cancel = vi.fn(async () => {});
fetchMock.mockResolvedValue({
ok: false,
status: 503,
body: { cancel },
} as any);
await expect(
searchDeveloperCategory(
{
query: "retries",
limit: 5,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
},
logger,
),
).resolves.toEqual([]);
expect(cancel).toHaveBeenCalledTimes(1);
});
it("returns no results when the upstream throws", async () => {
fetchMock.mockRejectedValue(new Error("connection reset"));
await expect(
searchDeveloperCategory(
{
query: "retries",
limit: 5,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
},
logger,
),
).resolves.toEqual([]);
});
});
describe("searchDeveloperCategory for a US-only team", () => {
const usOnlyOptions = {
query: "retries",
limit: 5,
teamId: "t1",
timeout: 500,
zeroDataRetention: false,
usOnly: true,
};
it("asks for US-only processing and reports the parts it skipped", async () => {
fetchMock.mockResolvedValue({
...upstreamOk({ results: [{ url: "https://github.com/a/b" }] }),
headers: new Headers({ "firecrawl-us-only-skipped": "web, docs" }),
});
const warnings: string[] = [];
const results = await searchDeveloperCategory(
{ ...usOnlyOptions, warnings },
logger,
);
expect(firstCall()[1].headers).toEqual({
"firecrawl-team-id": "t1",
"firecrawl-zero-data-retention": "false",
"firecrawl-us-only": "true",
});
expect(results.map(result => result.url)).toEqual([
"https://github.com/a/b",
]);
expect(warnings).toEqual([
"Some results were not available under US-only processing: web, docs.",
]);
});
it("relays an upstream refusal instead of returning no results", async () => {
fetchMock.mockResolvedValue({
ok: false,
status: 403,
json: async () => ({
error:
"Code search is not available for teams restricted to US-only processing.",
}),
} as any);
await expect(
searchDeveloperCategory(usOnlyOptions, logger),
).rejects.toMatchObject({
status: 403,
message:
"Code search is not available for teams restricted to US-only processing.",
});
});
});