Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): Jev-powered safe search filter for search safe: true (#4865)

Gergő Móriczbb3e582d7701
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 73a644bb1..e8a99fbc1 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -243,6 +243,15 @@ jobs:
run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=default
working-directory: apps/api
+ - name: Run search safe filter regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/search/safe-search.test.ts
+ src/search/execute.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run crawl status cursor regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: >-
diff --git a/apps/api/package.json b/apps/api/package.json
index b779f87ff..cad0c2262 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -94,6 +94,7 @@
"@opentelemetry/sdk-trace-node": "^2.11.0",
"@opentelemetry/semantic-conventions": "^1.43.0",
"@types/ws": "^8.5.12",
+ "@typesafe-ai/sdk": "^0.6.0",
"ai": "6.0.86",
"ajv": "^8.18.0",
"ajv-formats": "^3.0.1",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index beaff083d..e4f3bbed7 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -124,6 +124,9 @@ importers:
'@types/ws':
specifier: ^8.5.12
version: 8.5.12
+ '@typesafe-ai/sdk':
+ specifier: ^0.6.0
+ version: 0.6.0
ai:
specifier: 6.0.86
version: 6.0.86([email protected])
@@ -2013,6 +2016,10 @@ packages:
resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==}
+ '@typesafe-ai/[email protected]':
+ resolution: {integrity: sha512-IddX+Q0XM+VagOUZFeP7wZjaO4SHMdvnh2zEBdrZZnXedWI3BNK1lKhMx3ayrkFWvVLbVcUHJy6AVZlY+e6Jaw==}
+ engines: {node: '>=20'}
+
'@typescript/[email protected]':
resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==}
engines: {node: '>=16.20.0'}
@@ -6146,6 +6153,8 @@ snapshots:
dependencies:
'@types/node': 24.10.1
+ '@typesafe-ai/[email protected]': {}
+
'@typescript/[email protected]':
optional: true
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 6d1376b55..29870847c 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -315,6 +315,9 @@ const configSchema = z.object({
// returned. The remaining eligible traffic still runs in shadow mode.
HIGHLIGHT_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),
+ // TypeSafe (Jev): judges search results for the `safe: true` filter.
+ TYPESAFE_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
+
// Exchange (routed data sources service)
FIRE_EXCHANGE_URL: z.url().optional(),
EXCHANGE_INTERNAL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index bc30d9af6..34def52eb 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -3,6 +3,7 @@ const mocks = vi.hoisted(() => ({
searchDeveloperCategory: vi.fn(),
checkUrlsAgainstThreatPolicy: vi.fn(),
discoverTools: vi.fn(),
+ removeExplicitResults: vi.fn(),
}));
vi.mock("./alexandria", () => ({
@@ -42,6 +43,9 @@ vi.mock("../lib/threat-protection/request", () => ({
vi.mock("../lib/scrape-billing", () => ({
calculateThreatScanCredits: vi.fn(() => 0),
}));
+vi.mock("./safe-search", () => ({
+ removeExplicitResults: mocks.removeExplicitResults,
+}));
import { executeSearch } from "./execute";
import { searchRequestSchema } from "../controllers/v2/types";
@@ -116,6 +120,67 @@ it("mixed search bills only normal web results", async () => {
expect(result.response.tools).toHaveLength(1);
});
+describe("executeSearch safe search", () => {
+ const webResult = {
+ url: "https://example.com/",
+ title: "Example",
+ description: "An example page.",
+ };
+
+ it("bills and returns only the results the Jev filter keeps", async () => {
+ const safe = Array.from({ length: 10 }, (_, index) => ({
+ ...webResult,
+ url: `https://safe${index}.example/`,
+ }));
+ const explicit = { ...webResult, url: "https://nsfw.example/" };
+ mocks.search.mockResolvedValue({ web: [...safe, explicit] });
+ mocks.removeExplicitResults.mockImplementationOnce(async response => {
+ response.web = response.web.filter(
+ (result: { url: string }) => result.url !== explicit.url,
+ );
+ });
+
+ const result = await executeSearch(
+ { ...options([]), limit: 20, safe: true },
+ context,
+ logger,
+ );
+
+ expect(mocks.removeExplicitResults).toHaveBeenCalledWith(
+ expect.anything(),
+ 20,
+ logger,
+ );
+ expect(result.response.web?.map(x => x.url)).toEqual(safe.map(x => x.url));
+ expect(result.totalResultsCount).toBe(10);
+ // 11 results would bill 4 credits; the 10 kept bill 2.
+ expect(result.searchCredits).toBe(2);
+ });
+
+ it("skips the Jev filter when safe is off or data must not be retained", async () => {
+ mocks.search.mockResolvedValue({ web: [webResult] });
+
+ await executeSearch(options([]), context, logger);
+ await executeSearch(
+ { ...options([]), safe: true },
+ { ...context, zeroDataRetention: true },
+ logger,
+ );
+ await executeSearch(
+ { ...options([]), safe: true, enterprise: ["zdr"] },
+ context,
+ logger,
+ );
+ await executeSearch(
+ { ...options([]), safe: true, enterprise: ["anon"] },
+ context,
+ logger,
+ );
+
+ expect(mocks.removeExplicitResults).not.toHaveBeenCalled();
+ });
+});
+
describe("executeSearch developer category", () => {
it("returns sole developer-category results in web without running SERP", async () => {
const result = await executeSearch(
@@ -130,12 +195,29 @@ describe("executeSearch developer category", () => {
expect(result.developerResultsCount).toBe(1);
});
-
it("filters blocked developer results via threat protection and renumbers", async () => {
mocks.searchDeveloperCategory.mockResolvedValue([
- { url: "https://ok.example/a", title: "A", description: "", position: 1, category: "developer" },
- { url: "https://blocked.example/b", title: "B", description: "", position: 2, category: "developer" },
- { url: "https://ok.example/c", title: "C", description: "", position: 3, category: "developer" },
+ {
+ url: "https://ok.example/a",
+ title: "A",
+ description: "",
+ position: 1,
+ category: "developer",
+ },
+ {
+ url: "https://blocked.example/b",
+ title: "B",
+ description: "",
+ position: 2,
+ category: "developer",
+ },
+ {
+ url: "https://ok.example/c",
+ title: "C",
+ description: "",
+ position: 3,
+ category: "developer",
+ },
]);
mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({
decisionsByUrl: new Map([
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 07ded9c54..7689026cd 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -15,6 +15,7 @@ import {
calculateScrapeCredits,
} from "./scrape";
import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";
+import { removeExplicitResults } from "./safe-search";
import {
highlightsEnvReady,
runIndexedSearchHighlights,
@@ -203,6 +204,16 @@ export async function executeSearch(
}
}
+ // The filter shares results with TypeSafe, so zero data retention and
+ // anonymous requests skip it.
+ if (
+ options.safe &&
+ !zeroDataRetention &&
+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon")
+ ) {
+ await removeExplicitResults(searchResponse, limit, logger);
+ }
+
if (searchResponse.web && searchResponse.web.length > 0) {
searchResponse.web = searchResponse.web.map(result => ({
...result,
diff --git a/apps/api/src/search/safe-search.test.ts b/apps/api/src/search/safe-search.test.ts
new file mode 100644
index 000000000..9fb278b90
--- /dev/null
+++ b/apps/api/src/search/safe-search.test.ts
@@ -0,0 +1,205 @@
+const mocks = vi.hoisted(() => ({
+ systemOne: vi.fn(),
+ config: { TYPESAFE_API_KEY: "test-key" as string | undefined },
+}));
+
+vi.mock("@typesafe-ai/sdk", async importOriginal => ({
+ ...(await importOriginal<typeof import("@typesafe-ai/sdk")>()),
+ TypeSafeClient: class {
+ systemOne = mocks.systemOne;
+ },
+}));
+vi.mock("../config", () => ({ config: mocks.config }));
+
+import { removeExplicitResults } from "./safe-search";
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+} as any;
+
+const web = (name: string) => ({
+ url: `https://${name}.example/`,
+ title: name,
+ description: `${name} snippet`,
+});
+
+function judgeByUrl(explicitHosts: string[]) {
+ mocks.systemOne.mockImplementation(async ({ state }) => {
+ const url: string = state.result.url ?? "";
+ const explicit = explicitHosts.some(host => url.includes(`//${host}.`));
+ return {
+ answers: { explicit: { type: "noul", noul: explicit ? 0.9 : 0.1 } },
+ };
+ });
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.config.TYPESAFE_API_KEY = "test-key";
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+it("drops results Jev judges explicit and backfills from the surplus", async () => {
+ judgeByUrl(["nsfw1", "nsfw2"]);
+ const response = {
+ web: [
+ web("safe1"),
+ web("nsfw1"),
+ web("safe2"),
+ web("nsfw2"),
+ web("safe3"),
+ web("safe4"),
+ ],
+ };
+
+ await removeExplicitResults(response, 3, logger);
+
+ expect(response.web.map(result => result.title)).toEqual([
+ "safe1",
+ "safe2",
+ "safe3",
+ ]);
+ // Three judged first, then two to replace nsfw1 and nsfw2; safe4 is never needed.
+ expect(mocks.systemOne).toHaveBeenCalledTimes(5);
+ expect(mocks.systemOne).toHaveBeenCalledWith(
+ expect.objectContaining({
+ state: {
+ result: {
+ title: "safe1",
+ snippet: "safe1 snippet",
+ url: "https://safe1.example/",
+ },
+ },
+ }),
+ { signal: expect.any(AbortSignal) },
+ );
+});
+
+it("caps how many results are judged at once", async () => {
+ let inFlight = 0;
+ let maxInFlight = 0;
+ mocks.systemOne.mockImplementation(async () => {
+ inFlight++;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise(resolve => setTimeout(resolve, 1));
+ inFlight--;
+ return { answers: { explicit: { type: "noul", noul: 0.1 } } };
+ });
+ const response = {
+ web: Array.from({ length: 50 }, (_, index) => web(`safe${index}`)),
+ };
+
+ await removeExplicitResults(response, 50, logger);
+
+ expect(response.web).toHaveLength(50);
+ expect(mocks.systemOne).toHaveBeenCalledTimes(50);
+ expect(maxInFlight).toBe(20);
+});
+
+it("filters news and images alongside web", async () => {
+ judgeByUrl(["nsfw"]);
+ const response = {
+ news: [
+ { title: "News", url: "https://news.example/a", snippet: "ok" },
+ { title: "Bad", url: "https://nsfw.example/b", snippet: "bad" },
+ ],
+ images: [
+ {
+ title: "Bad",
+ url: "https://nsfw.example/c",
+ imageUrl: "https://cdn.example/c.jpg",
+ },
+ {
+ title: "Photo",
+ url: "https://photos.example/d",
+ imageUrl: "https://cdn.example/d.jpg",
+ },
+ ],
+ };
+
+ await removeExplicitResults(response, 5, logger);
+
+ expect(response.news.map(result => result.title)).toEqual(["News"]);
+ expect(response.images.map(result => result.title)).toEqual(["Photo"]);
+ expect(response).not.toHaveProperty("web");
+});
+
+it("keeps results Jev fails to judge", async () => {
+ mocks.systemOne.mockRejectedValue(new Error("upstream down"));
+ const response = { web: [web("a"), web("b")] };
+
+ await removeExplicitResults(response, 5, logger);
+
+ expect(response.web.map(result => result.title)).toEqual(["a", "b"]);
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Safe search filter kept results Jev could not judge",
+ expect.objectContaining({ failed: 2 }),
+ );
+});
+
+it("keeps results still pending when the filter's time budget runs out", async () => {
+ const budget = new AbortController();
+ const timeout = vi
+ .spyOn(AbortSignal, "timeout")
+ .mockReturnValue(budget.signal);
+ mocks.systemOne.mockImplementation(({ state }, { signal }) =>
+ state.result.url.includes("//nsfw.")
+ ? Promise.resolve({
+ answers: { explicit: { type: "noul", noul: 0.9 } },
+ })
+ : new Promise((_, reject) =>
+ signal.addEventListener("abort", () => reject(signal.reason)),
+ ),
+ );
+ const response = { web: [web("slow1"), web("nsfw"), web("slow2")] };
+
+ const run = removeExplicitResults(response, 5, logger);
+ await vi.waitFor(() => expect(mocks.systemOne).toHaveBeenCalledTimes(3));
+ budget.abort();
+ await run;
+ expect(timeout).toHaveBeenCalledWith(5000);
+
+ expect(response.web.map(result => result.title)).toEqual(["slow1", "slow2"]);
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Safe search filter kept results Jev could not judge",
+ expect.objectContaining({ failed: 2 }),
+ );
+});
+
+it("sends Jev only the start of ultralong fields", async () => {
+ judgeByUrl([]);
+ const response = {
+ web: [{ ...web("long"), description: "x".repeat(20_000) }],
+ };
+
+ await removeExplicitResults(response, 5, logger);
+
+ const [{ state }] = mocks.systemOne.mock.calls[0];
+ expect(state.result.snippet).toHaveLength(500);
+ expect(state.result.title).toBe("long");
+ expect(response.web[0].description).toHaveLength(20_000);
+});
+
+it("skips responses with nothing to judge", async () => {
+ await removeExplicitResults({}, 5, logger);
+ await removeExplicitResults({ web: [] }, 5, logger);
+
+ expect(mocks.systemOne).not.toHaveBeenCalled();
+ expect(logger.info).not.toHaveBeenCalled();
+});
+
+it("does nothing without a TypeSafe API key", async () => {
+ mocks.config.TYPESAFE_API_KEY = undefined;
+ const response = { web: [web("a")] };
+
+ await removeExplicitResults(response, 5, logger);
+
+ expect(mocks.systemOne).not.toHaveBeenCalled();
+ expect(response.web).toHaveLength(1);
+});
diff --git a/apps/api/src/search/safe-search.ts b/apps/api/src/search/safe-search.ts
new file mode 100644
index 000000000..cb7cc4304
--- /dev/null
+++ b/apps/api/src/search/safe-search.ts
@@ -0,0 +1,176 @@
+import { noul, TypeSafeClient, type JsonValue } from "@typesafe-ai/sdk";
+import type { Logger } from "winston";
+import type { SearchV2Response } from "../lib/entities";
+import { config } from "../config";
+import { setSpanAttributes, SpanKind, withSpan } from "../lib/otel-tracer";
+
+/**
+ * Jev's probability that a result is adult content, above which it is dropped.
+ * Labeled adult results scored 0.42 and up, safe ones 0.04 and below.
+ */
+const EXPLICIT_THRESHOLD = 0.3;
+
+/** Most Jev calls in flight at once for one result type. */
+const MAX_CONCURRENT_JUDGMENTS = 20;
+
+/** Past this, results not yet judged are kept rather than delaying the search. */
+const FILTER_BUDGET_MS = 5000;
+
+/** Some providers return page markdown as the snippet; Jev only needs the start. */
+const MAX_FIELD_CHARS = 500;
+
+const clip = (text: string | undefined): string | null =>
+ text ? text.slice(0, MAX_FIELD_CHARS) : null;
+
+const questions = {
+ explicit: noul(
+ {
+ task: "`result` is one result a web search returned. Is the result adult content, or does it mention an adult content platform?",
+ guidance:
+ "Judge the result's own title, snippet and URL. Any mention of an adult content platform counts, in any context, including news, business, software, tax or creator advice.",
+ },
+ {
+ true: "Pornographic or sexually explicit material; NSFW or sex AI generators and chat; escorting or camming; or any mention of an adult content platform or adult creator subscription site such as OnlyFans, Fansly, ManyVids, Chaturbate or Pornhub, including guides, comparisons, clones and news about them.",
+ false:
+ "Anything else, including sex education, sexual health and medicine, dating, fashion, and uses of 'adult' that mean grown-up, such as adult education or adult ADHD.",
+ },
+ ),
+};
+
+let client: TypeSafeClient | undefined;
+
+function getClient(): TypeSafeClient | null {
+ if (!config.TYPESAFE_API_KEY) return null;
+ client ??= new TypeSafeClient({
+ apiKey: config.TYPESAFE_API_KEY,
+ timeout: 2000,
+ retry: { maxRetries: 1 },
+ logLevel: "off",
+ });
+ return client;
+}
+
+/**
+ * Drops web, news and image results Jev judges to be adult content, keeping up
+ * to `limit` of each in their original order. A result Jev fails to judge is
+ * kept, so an outage or slow judgment falls back to the search provider's own
+ * safe search.
+ */
+export async function removeExplicitResults(
+ response: SearchV2Response,
+ limit: number,
+ logger: Logger,
+): Promise<void> {
+ const { web, news, images } = response;
+ if (!web?.length && !news?.length && !images?.length) return;
+
+ const typesafe = getClient();
+ if (!typesafe) return;
+
+ await withSpan(
+ "search.safe_filter",
+ async span => {
+ const signal = AbortSignal.timeout(FILTER_BUDGET_MS);
+ let judged = 0;
+ let dropped = 0;
+ let failed = 0;
+ let lastError: unknown;
+
+ const isExplicit = async (
+ type: string,
+ result: Record<string, JsonValue>,
+ ): Promise<boolean> => {
+ judged++;
+ try {
+ return await withSpan(
+ "typesafe.systemone",
+ async callSpan => {
+ const { model, answers } = await typesafe.systemOne(
+ { state: { result }, questions },
+ { signal },
+ );
+ setSpanAttributes(callSpan, {
+ "typesafe.model": model,
+ "search.safe_filter.explicit_probability":
+ answers.explicit.noul,
+ });
+ return answers.explicit.noul > EXPLICIT_THRESHOLD;
+ },
+ {
+ kind: SpanKind.CLIENT,
+ attributes: { "search.safe_filter.result_type": type },
+ },
+ );
+ } catch (error) {
+ failed++;
+ lastError = error;
+ return false;
+ }
+ };
+
+ // Judges only as many results as can still be returned, then backfills
+ // from the provider's surplus for each one dropped.
+ const keepSafe = async <T>(
+ type: string,
+ items: T[],
+ describe: (item: T) => Record<string, JsonValue>,
+ ): Promise<T[]> => {
+ const kept: T[] = [];
+ let next = 0;
+ while (kept.length < limit && next < items.length) {
+ const batch = items.slice(
+ next,
+ next + Math.min(limit - kept.length, MAX_CONCURRENT_JUDGMENTS),
+ );
+ next += batch.length;
+ const verdicts = await Promise.all(
+ batch.map(item => isExplicit(type, describe(item))),
+ );
+ batch.forEach((item, index) => {
+ if (verdicts[index]) dropped++;
+ else kept.push(item);
+ });
+ }
+ return kept;
+ };
+
+ const [safeWeb, safeNews, safeImages] = await Promise.all([
+ web &&
+ keepSafe("web", web, result => ({
+ title: clip(result.title),
+ snippet: clip(result.description),
+ url: clip(result.url),
+ })),
+ news &&
+ keepSafe("news", news, result => ({
+ title: clip(result.title),
+ snippet: clip(result.snippet),
+ url: clip(result.url),
+ })),
+ images &&
+ keepSafe("images", images, result => ({
+ title: clip(result.title),
+ url: clip(result.url),
+ imageUrl: clip(result.imageUrl),
+ })),
+ ]);
+ if (safeWeb) response.web = safeWeb;
+ if (safeNews) response.news = safeNews;
+ if (safeImages) response.images = safeImages;
+
+ setSpanAttributes(span, {
+ "search.safe_filter.judged": judged,
+ "search.safe_filter.dropped": dropped,
+ "search.safe_filter.failed": failed,
+ });
+ logger.info("Safe search filter applied", { judged, dropped, failed });
+ if (failed > 0) {
+ logger.warn("Safe search filter kept results Jev could not judge", {
+ failed,
+ error: lastError,
+ });
+ }
+ },
+ { attributes: { "search.safe_filter.limit": limit } },
+ );
+}