Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): Jev-powered safe search filter for search safe: true (#4865)
Gergő Móriczbb3e582d7701
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 73a644bb1..e8a99fbc1 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -243,6 +243,15 @@ jobs:run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run search safe filter regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/search/safe-search.test.ts+ src/search/execute.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run crawl status cursor regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: >-diff --git a/apps/api/package.json b/apps/api/package.jsonindex b779f87ff..cad0c2262 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -94,6 +94,7 @@"@opentelemetry/sdk-trace-node": "^2.11.0","@opentelemetry/semantic-conventions": "^1.43.0","@types/ws": "^8.5.12",+ "@typesafe-ai/sdk": "^0.6.0","ai": "6.0.86","ajv": "^8.18.0","ajv-formats": "^3.0.1",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex beaff083d..e4f3bbed7 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -124,6 +124,9 @@ importers:'@types/ws':specifier: ^8.5.12version: 8.5.12+ '@typesafe-ai/sdk':+ specifier: ^0.6.0+ version: 0.6.0ai:specifier: 6.0.86version: 6.0.86([email protected])@@ -2013,6 +2016,10 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==}+ '@typesafe-ai/[email protected]':+ resolution: {integrity: sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==}+ engines: {node: '>=20'}+'@typescript/[email protected]':resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==}engines: {node: '>=16.20.0'}@@ -6146,6 +6153,8 @@ snapshots:dependencies:'@types/node': 24.10.1+ '@typesafe-ai/[email protected]': {}+'@typescript/[email protected]':optional: truediff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 6d1376b55..29870847c 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -315,6 +315,9 @@ const configSchema = z.object({// returned. The remaining eligible traffic still runs in shadow mode.HIGHLIGHT_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),+ // TypeSafe (Jev): judges search results for the `safe: true` filter.+ TYPESAFE_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),+// Exchange (routed data sources service)FIRE_EXCHANGE_URL: z.url().optional(),EXCHANGE_INTERNAL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex bc30d9af6..34def52eb 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -3,6 +3,7 @@ const mocks = vi.hoisted(() => ({searchDeveloperCategory: vi.fn(),checkUrlsAgainstThreatPolicy: vi.fn(),discoverTools: vi.fn(),+ removeExplicitResults: vi.fn(),}));vi.mock("./alexandria", () => ({@@ -42,6 +43,9 @@ vi.mock("../lib/threat-protection/request", () => ({vi.mock("../lib/scrape-billing", () => ({calculateThreatScanCredits: vi.fn(() => 0),}));+vi.mock("./safe-search", () => ({+ removeExplicitResults: mocks.removeExplicitResults,+}));import { executeSearch } from "./execute";import { searchRequestSchema } from "../controllers/v2/types";@@ -116,6 +120,67 @@ it("mixed search bills only normal web results", async () => {expect(result.response.tools).toHaveLength(1);});+describe("executeSearch safe search", () => {+ const webResult = {+ url: "https://example.com/",+ title: "Example",+ description: "An example page.",+ };++ it("bills and returns only the results the Jev filter keeps", async () => {+ const safe = Array.from({ length: 10 }, (_, index) => ({+ ...webResult,+ url: `https://safe${index}.example/`,+ }));+ const explicit = { ...webResult, url: "https://nsfw.example/" };+ mocks.search.mockResolvedValue({ web: [...safe, explicit] });+ mocks.removeExplicitResults.mockImplementationOnce(async response => {+ response.web = response.web.filter(+ (result: { url: string }) => result.url !== explicit.url,+ );+ });++ const result = await executeSearch(+ { ...options([]), limit: 20, safe: true },+ context,+ logger,+ );++ expect(mocks.removeExplicitResults).toHaveBeenCalledWith(+ expect.anything(),+ 20,+ logger,+ );+ expect(result.response.web?.map(x => x.url)).toEqual(safe.map(x => x.url));+ expect(result.totalResultsCount).toBe(10);+ // 11 results would bill 4 credits; the 10 kept bill 2.+ expect(result.searchCredits).toBe(2);+ });++ it("skips the Jev filter when safe is off or data must not be retained", async () => {+ mocks.search.mockResolvedValue({ web: [webResult] });++ await executeSearch(options([]), context, logger);+ await executeSearch(+ { ...options([]), safe: true },+ { ...context, zeroDataRetention: true },+ logger,+ );+ await executeSearch(+ { ...options([]), safe: true, enterprise: ["zdr"] },+ context,+ logger,+ );+ await executeSearch(+ { ...options([]), safe: true, enterprise: ["anon"] },+ context,+ logger,+ );++ expect(mocks.removeExplicitResults).not.toHaveBeenCalled();+ });+});+describe("executeSearch developer category", () => {it("returns sole developer-category results in web without running SERP", async () => {const result = await executeSearch(@@ -130,12 +195,29 @@ describe("executeSearch developer category", () => {expect(result.developerResultsCount).toBe(1);});-it("filters blocked developer results via threat protection and renumbers", async () => {mocks.searchDeveloperCategory.mockResolvedValue([- { url: "https://ok.example/a", title: "A", description: "", position: 1, category: "developer" },- { url: "https://blocked.example/b", title: "B", description: "", position: 2, category: "developer" },- { url: "https://ok.example/c", title: "C", description: "", position: 3, category: "developer" },+ {+ url: "https://ok.example/a",+ title: "A",+ description: "",+ position: 1,+ category: "developer",+ },+ {+ url: "https://blocked.example/b",+ title: "B",+ description: "",+ position: 2,+ category: "developer",+ },+ {+ url: "https://ok.example/c",+ title: "C",+ description: "",+ position: 3,+ category: "developer",+ },]);mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({decisionsByUrl: new Map([diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 07ded9c54..7689026cd 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -15,6 +15,7 @@ import {calculateScrapeCredits,} from "./scrape";import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";+import { removeExplicitResults } from "./safe-search";import {highlightsEnvReady,runIndexedSearchHighlights,@@ -203,6 +204,16 @@ export async function executeSearch(}}+ // The filter shares results with TypeSafe, so zero data retention and+ // anonymous requests skip it.+ if (+ options.safe &&+ !zeroDataRetention &&+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon")+ ) {+ await removeExplicitResults(searchResponse, limit, logger);+ }+if (searchResponse.web && searchResponse.web.length > 0) {searchResponse.web = searchResponse.web.map(result => ({...result,diff --git a/apps/api/src/search/safe-search.test.ts b/apps/api/src/search/safe-search.test.tsnew file mode 100644index 000000000..9fb278b90--- /dev/null+++ b/apps/api/src/search/safe-search.test.ts@@ -0,0 +1,205 @@+const mocks = vi.hoisted(() => ({+ systemOne: vi.fn(),+ config: { TYPESAFE_API_KEY: "test-key" as string | undefined },+}));++vi.mock("@typesafe-ai/sdk", async importOriginal => ({+ ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),+ TypeSafeClient: class {+ systemOne = mocks.systemOne;+ },+}));+vi.mock("../config", () => ({ config: mocks.config }));++import { removeExplicitResults } from "./safe-search";++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+} as any;++const web = (name: string) => ({+ url: `https://${name}.example/`,+ title: name,+ description: `${name} snippet`,+});++function judgeByUrl(explicitHosts: string[]) {+ mocks.systemOne.mockImplementation(async ({ state }) => {+ const url: string = state.result.url ?? "";+ const explicit = explicitHosts.some(host => url.includes(`//${host}.`));+ return {+ answers: { explicit: { type: "noul", noul: explicit ? 0.9 : 0.1 } },+ };+ });+}++beforeEach(() => {+ vi.clearAllMocks();+ mocks.config.TYPESAFE_API_KEY = "test-key";+});++afterEach(() => {+ vi.restoreAllMocks();+});++it("drops results Jev judges explicit and backfills from the surplus", async () => {+ judgeByUrl(["nsfw1", "nsfw2"]);+ const response = {+ web: [+ web("safe1"),+ web("nsfw1"),+ web("safe2"),+ web("nsfw2"),+ web("safe3"),+ web("safe4"),+ ],+ };++ await removeExplicitResults(response, 3, logger);++ expect(response.web.map(result => result.title)).toEqual([+ "safe1",+ "safe2",+ "safe3",+ ]);+ // Three judged first, then two to replace nsfw1 and nsfw2; safe4 is never needed.+ expect(mocks.systemOne).toHaveBeenCalledTimes(5);+ expect(mocks.systemOne).toHaveBeenCalledWith(+ expect.objectContaining({+ state: {+ result: {+ title: "safe1",+ snippet: "safe1 snippet",+ url: "https://safe1.example/",+ },+ },+ }),+ { signal: expect.any(AbortSignal) },+ );+});++it("caps how many results are judged at once", async () => {+ let inFlight = 0;+ let maxInFlight = 0;+ mocks.systemOne.mockImplementation(async () => {+ inFlight++;+ maxInFlight = Math.max(maxInFlight, inFlight);+ await new Promise(resolve => setTimeout(resolve, 1));+ inFlight--;+ return { answers: { explicit: { type: "noul", noul: 0.1 } } };+ });+ const response = {+ web: Array.from({ length: 50 }, (_, index) => web(`safe${index}`)),+ };++ await removeExplicitResults(response, 50, logger);++ expect(response.web).toHaveLength(50);+ expect(mocks.systemOne).toHaveBeenCalledTimes(50);+ expect(maxInFlight).toBe(20);+});++it("filters news and images alongside web", async () => {+ judgeByUrl(["nsfw"]);+ const response = {+ news: [+ { title: "News", url: "https://news.example/a", snippet: "ok" },+ { title: "Bad", url: "https://nsfw.example/b", snippet: "bad" },+ ],+ images: [+ {+ title: "Bad",+ url: "https://nsfw.example/c",+ imageUrl: "https://cdn.example/c.jpg",+ },+ {+ title: "Photo",+ url: "https://photos.example/d",+ imageUrl: "https://cdn.example/d.jpg",+ },+ ],+ };++ await removeExplicitResults(response, 5, logger);++ expect(response.news.map(result => result.title)).toEqual(["News"]);+ expect(response.images.map(result => result.title)).toEqual(["Photo"]);+ expect(response).not.toHaveProperty("web");+});++it("keeps results Jev fails to judge", async () => {+ mocks.systemOne.mockRejectedValue(new Error("upstream down"));+ const response = { web: [web("a"), web("b")] };++ await removeExplicitResults(response, 5, logger);++ expect(response.web.map(result => result.title)).toEqual(["a", "b"]);+ expect(logger.warn).toHaveBeenCalledWith(+ "Safe search filter kept results Jev could not judge",+ expect.objectContaining({ failed: 2 }),+ );+});++it("keeps results still pending when the filter's time budget runs out", async () => {+ const budget = new AbortController();+ const timeout = vi+ .spyOn(AbortSignal, "timeout")+ .mockReturnValue(budget.signal);+ mocks.systemOne.mockImplementation(({ state }, { signal }) =>+ state.result.url.includes("//nsfw.")+ ? Promise.resolve({+ answers: { explicit: { type: "noul", noul: 0.9 } },+ })+ : new Promise((_, reject) =>+ signal.addEventListener("abort", () => reject(signal.reason)),+ ),+ );+ const response = { web: [web("slow1"), web("nsfw"), web("slow2")] };++ const run = removeExplicitResults(response, 5, logger);+ await vi.waitFor(() => expect(mocks.systemOne).toHaveBeenCalledTimes(3));+ budget.abort();+ await run;+ expect(timeout).toHaveBeenCalledWith(5000);++ expect(response.web.map(result => result.title)).toEqual(["slow1", "slow2"]);+ expect(logger.warn).toHaveBeenCalledWith(+ "Safe search filter kept results Jev could not judge",+ expect.objectContaining({ failed: 2 }),+ );+});++it("sends Jev only the start of ultralong fields", async () => {+ judgeByUrl([]);+ const response = {+ web: [{ ...web("long"), description: "x".repeat(20_000) }],+ };++ await removeExplicitResults(response, 5, logger);++ const [{ state }] = mocks.systemOne.mock.calls[0];+ expect(state.result.snippet).toHaveLength(500);+ expect(state.result.title).toBe("long");+ expect(response.web[0].description).toHaveLength(20_000);+});++it("skips responses with nothing to judge", async () => {+ await removeExplicitResults({}, 5, logger);+ await removeExplicitResults({ web: [] }, 5, logger);++ expect(mocks.systemOne).not.toHaveBeenCalled();+ expect(logger.info).not.toHaveBeenCalled();+});++it("does nothing without a TypeSafe API key", async () => {+ mocks.config.TYPESAFE_API_KEY = undefined;+ const response = { web: [web("a")] };++ await removeExplicitResults(response, 5, logger);++ expect(mocks.systemOne).not.toHaveBeenCalled();+ expect(response.web).toHaveLength(1);+});diff --git a/apps/api/src/search/safe-search.ts b/apps/api/src/search/safe-search.tsnew file mode 100644index 000000000..cb7cc4304--- /dev/null+++ b/apps/api/src/search/safe-search.ts@@ -0,0 +1,176 @@+import { noul, TypeSafeClient, type JsonValue } from "@typesafe-ai/sdk";+import type { Logger } from "winston";+import type { SearchV2Response } from "../lib/entities";+import { config } from "../config";+import { setSpanAttributes, SpanKind, withSpan } from "../lib/otel-tracer";++/**+ * Jev's probability that a result is adult content, above which it is dropped.+ * Labeled adult results scored 0.42 and up, safe ones 0.04 and below.+ */+const EXPLICIT_THRESHOLD = 0.3;++/** Most Jev calls in flight at once for one result type. */+const MAX_CONCURRENT_JUDGMENTS = 20;++/** Past this, results not yet judged are kept rather than delaying the search. */+const FILTER_BUDGET_MS = 5000;++/** Some providers return page markdown as the snippet; Jev only needs the start. */+const MAX_FIELD_CHARS = 500;++const clip = (text: string | undefined): string | null =>+ text ? text.slice(0, MAX_FIELD_CHARS) : null;++const questions = {+ explicit: noul(+ {+ task: "`result` is one result a web search returned. Is the result adult content, or does it mention an adult content platform?",+ guidance:+ "Judge the result's own title, snippet and URL. Any mention of an adult content platform counts, in any context, including news, business, software, tax or creator advice.",+ },+ {+ true: "Pornographic or sexually explicit material; NSFW or sex AI generators and chat; escorting or camming; or any mention of an adult content platform or adult creator subscription site such as OnlyFans, Fansly, ManyVids, Chaturbate or Pornhub, including guides, comparisons, clones and news about them.",+ false:+ "Anything else, including sex education, sexual health and medicine, dating, fashion, and uses of 'adult' that mean grown-up, such as adult education or adult ADHD.",+ },+ ),+};++let client: TypeSafeClient | undefined;++function getClient(): TypeSafeClient | null {+ if (!config.TYPESAFE_API_KEY) return null;+ client ??= new TypeSafeClient({+ apiKey: config.TYPESAFE_API_KEY,+ timeout: 2000,+ retry: { maxRetries: 1 },+ logLevel: "off",+ });+ return client;+}++/**+ * Drops web, news and image results Jev judges to be adult content, keeping up+ * to `limit` of each in their original order. A result Jev fails to judge is+ * kept, so an outage or slow judgment falls back to the search provider's own+ * safe search.+ */+export async function removeExplicitResults(+ response: SearchV2Response,+ limit: number,+ logger: Logger,+): Promise<void> {+ const { web, news, images } = response;+ if (!web?.length && !news?.length && !images?.length) return;++ const typesafe = getClient();+ if (!typesafe) return;++ await withSpan(+ "search.safe_filter",+ async span => {+ const signal = AbortSignal.timeout(FILTER_BUDGET_MS);+ let judged = 0;+ let dropped = 0;+ let failed = 0;+ let lastError: unknown;++ const isExplicit = async (+ type: string,+ result: Record<string, JsonValue>,+ ): Promise<boolean> => {+ judged++;+ try {+ return await withSpan(+ "typesafe.systemone",+ async callSpan => {+ const { model, answers } = await typesafe.systemOne(+ { state: { result }, questions },+ { signal },+ );+ setSpanAttributes(callSpan, {+ "typesafe.model": model,+ "search.safe_filter.explicit_probability":+ answers.explicit.noul,+ });+ return answers.explicit.noul > EXPLICIT_THRESHOLD;+ },+ {+ kind: SpanKind.CLIENT,+ attributes: { "search.safe_filter.result_type": type },+ },+ );+ } catch (error) {+ failed++;+ lastError = error;+ return false;+ }+ };++ // Judges only as many results as can still be returned, then backfills+ // from the provider's surplus for each one dropped.+ const keepSafe = async <T>(+ type: string,+ items: T[],+ describe: (item: T) => Record<string, JsonValue>,+ ): Promise<T[]> => {+ const kept: T[] = [];+ let next = 0;+ while (kept.length < limit && next < items.length) {+ const batch = items.slice(+ next,+ next + Math.min(limit - kept.length, MAX_CONCURRENT_JUDGMENTS),+ );+ next += batch.length;+ const verdicts = await Promise.all(+ batch.map(item => isExplicit(type, describe(item))),+ );+ batch.forEach((item, index) => {+ if (verdicts[index]) dropped++;+ else kept.push(item);+ });+ }+ return kept;+ };++ const [safeWeb, safeNews, safeImages] = await Promise.all([+ web &&+ keepSafe("web", web, result => ({+ title: clip(result.title),+ snippet: clip(result.description),+ url: clip(result.url),+ })),+ news &&+ keepSafe("news", news, result => ({+ title: clip(result.title),+ snippet: clip(result.snippet),+ url: clip(result.url),+ })),+ images &&+ keepSafe("images", images, result => ({+ title: clip(result.title),+ url: clip(result.url),+ imageUrl: clip(result.imageUrl),+ })),+ ]);+ if (safeWeb) response.web = safeWeb;+ if (safeNews) response.news = safeNews;+ if (safeImages) response.images = safeImages;++ setSpanAttributes(span, {+ "search.safe_filter.judged": judged,+ "search.safe_filter.dropped": dropped,+ "search.safe_filter.failed": failed,+ });+ logger.info("Safe search filter applied", { judged, dropped, failed });+ if (failed > 0) {+ logger.warn("Safe search filter kept results Jev could not judge", {+ failed,+ error: lastError,+ });+ }+ },+ { attributes: { "search.safe_filter.limit": limit } },+ );+}