Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/search): enforce site:, inurl: and intitle: operator combinations (#5381)
Rakshith Ramprakash13673be17227
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 1785e3412..b3d9dd007 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -616,3 +616,214 @@ describe("executeSearch gov category", () => {expect(result.searchCredits).toBe(0);});});++describe("executeSearch query operators", () => {+ const web = (url: string, title = "Result") => ({+ url,+ title,+ description: "",+ });+ const news = (url: string, title: string) => ({ url, title, snippet: "" });++ it("sends inurl: with site: as plain terms and enforces both on web results", async () => {+ mocks.search.mockResolvedValue({+ web: [+ web(+ "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",+ ),+ web("https://www.finance.example/quotes/CTSO"),+ web("https://www.stocks.example/stocks/xnas/ctso/earnings"),+ web(+ "https://www.finance.example/2026/04/29/contoso-ctso-q3-earnings-report-2026.html",+ ),+ ],+ });++ const result = await executeSearch(+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },+ context,+ logger,+ );++ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({+ query: "earnings Contoso site:finance.example",+ includeDomains: ["finance.example"],+ }),+ );+ expect(result.response.web?.map(x => [x.position, x.url])).toEqual([+ [+ 1,+ "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",+ ],+ [+ 2,+ "https://www.finance.example/2026/04/29/contoso-ctso-q3-earnings-report-2026.html",+ ],+ ]);+ });++ it("bills nothing when the operators leave no results", async () => {+ mocks.search.mockResolvedValue({+ web: [web("https://www.finance.example/quotes/CTSO")],+ });++ const result = await executeSearch(+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },+ context,+ logger,+ );++ expect(result.response.web).toEqual([]);+ expect(result.totalResultsCount).toBe(0);+ expect(result.searchCredits).toBe(0);+ });++ it("applies both intitle: and site: to news results", async () => {+ mocks.search.mockResolvedValue({+ news: [+ news(+ "https://www.news.example/aktuell/wirtschaft/a.html",+ "Acme: Quarterly results - News",+ ),+ news(+ "https://www.forum.example/r/insurance/comments/1/acme/",+ "Acme denied claim",+ ),+ news(+ "https://www.news.example/agenturmeldungen/dpa/b.html",+ "Local team wins again - News",+ ),+ ],+ });++ const result = await executeSearch(+ {+ ...options([]),+ query: '"Acme" intitle:Acme site:news.example',+ sources: [{ type: "news" }],+ tbs: "qdr:w",+ },+ context,+ logger,+ );++ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({+ query: '"Acme" Acme site:news.example',+ includeDomains: ["news.example"],+ tbs: "qdr:w",+ }),+ );+ expect(result.response.news?.map(x => x.url)).toEqual([+ "https://www.news.example/aktuell/wirtschaft/a.html",+ ]);+ });++ it("enforces site: alone on news results", async () => {+ mocks.search.mockResolvedValue({+ news: [+ news("https://www.news.example/aktuell/a.html", "A"),+ news("https://www.acmecare.example/en/blog.html", "B"),+ ],+ });++ const result = await executeSearch(+ {+ ...options([]),+ query: "Acme site:news.example",+ sources: [{ type: "news" }],+ },+ context,+ logger,+ );++ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ query: "Acme site:news.example" }),+ );+ expect(result.response.news?.map(x => x.url)).toEqual([+ "https://www.news.example/aktuell/a.html",+ ]);+ });++ it("enforces site: on image results", async () => {+ mocks.search.mockResolvedValue({+ images: [+ {+ url: "https://www.news.example/a.html",+ title: "A",+ imageUrl: "https://cdn.example/a.jpg",+ },+ {+ url: "https://other.example/b.html",+ title: "B",+ imageUrl: "https://cdn.example/b.jpg",+ },+ ],+ });++ const result = await executeSearch(+ {+ ...options([]),+ query: "Acme site:news.example",+ sources: [{ type: "images" }],+ },+ context,+ logger,+ );++ expect(result.response.images?.map(x => [x.position, x.url])).toEqual([+ [1, "https://www.news.example/a.html"],+ ]);+ });++ it("enforces site: on category index results", async () => {+ mocks.searchDeveloperCategory.mockResolvedValue([+ developerResult,+ { ...developerResult, url: "https://docs.example/retries", position: 2 },+ ]);++ const result = await executeSearch(+ {+ ...options([{ type: "developer" }]),+ query: "retries site:docs.example",+ },+ context,+ logger,+ );++ expect(result.response.web?.map(x => [x.position, x.url])).toEqual([+ [1, "https://docs.example/retries"],+ ]);+ expect(result.indexResultsCount).toBe(1);+ });++ it("asks the provider for more results when it enforces inurl: or intitle: itself", async () => {+ mocks.search.mockResolvedValue({ web: [] });++ await executeSearch(+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },+ context,+ logger,+ );++ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ num_results: 50 }),+ );+ });++ it("forwards a custom date range tbs to the web provider unchanged", async () => {+ mocks.search.mockResolvedValue({ web: [] });+ const tbs = "cdr:1,cd_min:1/1/2024,cd_max:1/31/2024";++ await executeSearch(+ { ...options([]), query: "Contoso earnings site:finance.example", tbs },+ context,+ logger,+ );++ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ tbs, type: ["web"] }),+ );+ });+});diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 484f535f9..40548fd3e 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -29,6 +29,11 @@ import {import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";import { searchGovCategory, wantsGovCategory } from "./gov";import { removeExplicitResults } from "./safe-search";+import {+ matchesQueryOperators,+ planOperatorQuery,+ siteHostnames,+} from "./operators";import {highlightsEnvReady,runIndexedSearchHighlights,@@ -190,8 +195,11 @@ export async function executeSearch(.map(source => source.type as SearchResultType),),];+ const operatorPlan = planOperatorQuery(query);+ const { operators } = operatorPlan;+ const nonEmpty = (list: string[]) => (list.length > 0 ? list : undefined);const { query: searchQuery, categoryMap } = buildSearchQuery(- query,+ operatorPlan.query,categories,{includeDomains: options.includeDomains,@@ -231,7 +239,9 @@ export async function executeSearch(logger,requestId: context.requestId,advanced: false,- num_results: num_results_buffer,+ num_results: operatorPlan.relaxed+ ? Math.max(num_results_buffer, Math.min(limit * 5, 100))+ : num_results_buffer,tbs: options.tbs,filter: options.filter,lang: options.lang,@@ -240,10 +250,33 @@ export async function executeSearch(safe: options.safe,type: searchTypes,enterprise: options.enterprise,- includeDomains: options.includeDomains,- excludeDomains: options.excludeDomains,+ includeDomains: options.includeDomains?.length+ ? options.includeDomains+ : nonEmpty(siteHostnames(operators.sites)),+ excludeDomains: nonEmpty([+ ...(options.excludeDomains ?? []),+ ...operators.excludedSites.filter(site => !site.includes("/")),+ ]),})) as SearchV2Response);- let indexResults = indexResultsPromise ? await indexResultsPromise : [];+ const keepOperatorMatches = <+ T extends { url?: string; title?: string; position?: number },+ >(+ results: T[] | undefined,+ ): T[] | undefined => {+ const kept = results?.filter(result =>+ matchesQueryOperators(result, operatorPlan),+ );+ return kept?.length === results?.length+ ? results+ : kept?.map((result, index) => ({ ...result, position: index + 1 }));+ };+ searchResponse.web = keepOperatorMatches(searchResponse.web);+ searchResponse.news = keepOperatorMatches(searchResponse.news);+ searchResponse.images = keepOperatorMatches(searchResponse.images);+ let indexResults =+ keepOperatorMatches(+ indexResultsPromise ? await indexResultsPromise : undefined,+ ) ?? [];// Threat protection: remove blocked results entirely — before// slicing/counting, before scraping, and before returning. Checks arediff --git a/apps/api/src/search/operators.test.ts b/apps/api/src/search/operators.test.tsnew file mode 100644index 000000000..29c5d1056--- /dev/null+++ b/apps/api/src/search/operators.test.ts@@ -0,0 +1,218 @@+import { describe, expect, it } from "vitest";+import {+ matchesQueryOperators,+ parseQueryOperators,+ planOperatorQuery,+ siteHostnames,+} from "./operators";++describe("parseQueryOperators", () => {+ it("reads site:, -site:, inurl:, intitle: and quoted values", () => {+ expect(+ parseQueryOperators(+ '"Acme" intitle:"Acme Corp" inurl:earnings -inurl:video site:news.example -site:jobs.news.example',+ ),+ ).toEqual({+ sites: ["news.example"],+ excludedSites: ["jobs.news.example"],+ inurl: ["earnings"],+ intitle: ["acme corp"],+ excludedInurl: ["video"],+ excludedIntitle: [],+ });+ });++ it("reads site: inside an OR group and ignores allintitle:", () => {+ const operators = parseQueryOperators(+ "(site:a.com OR site:https://b.com/docs/) allintitle:foo",+ );+ expect(operators.sites).toEqual(["a.com", "b.com/docs"]);+ expect(operators.intitle).toEqual([]);+ });+});++describe("planOperatorQuery", () => {+ it("ignores operator-like text inside quoted phrases", () => {+ const plan = planOperatorQuery('"Acme inurl:earnings" site:news.example');+ expect(plan.operators.inurl).toEqual([]);+ expect(plan.relaxed).toBe(false);+ expect(plan.query).toBe('"Acme inurl:earnings" site:news.example');+ });++ it("enforces nothing when OR joins anything but site: operators", () => {+ for (const query of [+ "(site:news.example OR intitle:Acme)",+ "Acme site:news.example OR earnings",+ ]) {+ const plan = planOperatorQuery(query);+ expect(plan.query).toBe(query);+ expect(plan.relaxed).toBe(false);+ expect(plan.operators.sites).toEqual([]);+ expect(+ matchesQueryOperators({ url: "https://other.example/" }, plan),+ ).toBe(true);+ }+ });++ it("keeps an OR chain of site: operators", () => {+ expect(+ planOperatorQuery(+ "Acme (site:a.example OR site:b.example OR site:c.example)",+ ).operators.sites,+ ).toEqual(["a.example", "b.example", "c.example"]);+ });++ it("keeps the closing parenthesis of a dropped negated operator", () => {+ expect(+ planOperatorQuery("site:news.example (Acme -inurl:video)").query,+ ).toBe("site:news.example (Acme )");+ });++ it("normalizes unicode site: hosts to ASCII", () => {+ const plan = planOperatorQuery("Acme site:bücher.example");+ expect(plan.operators.sites).toEqual(["xn--bcher-kva.example"]);+ expect(+ matchesQueryOperators({ url: "https://www.bücher.example/a" }, plan),+ ).toBe(true);+ });++ it("keeps a query without the site: plus inurl:/intitle: combination", () => {+ for (const query of [+ "earnings Contoso site:finance.example",+ "inurl:earnings Contoso",+ "intitle:Acme",+ ]) {+ const plan = planOperatorQuery(query);+ expect(plan.query).toBe(query);+ expect(plan.relaxed).toBe(false);+ }+ });++ it("sends inurl: with site: as a plain term", () => {+ const plan = planOperatorQuery(+ "inurl:earnings Contoso site:finance.example",+ );+ expect(plan.relaxed).toBe(true);+ expect(plan.query).toBe("earnings Contoso site:finance.example");+ });++ it("sends intitle: with site: as a plain term and enforces negated filters on results", () => {+ const plan = planOperatorQuery(+ '"Acme" intitle:Acme -intitle:Stadium site:news.example',+ );+ expect(plan.query).toBe('"Acme" Acme site:news.example');+ expect(+ matchesQueryOperators(+ { url: "https://www.news.example/a", title: "Acme results" },+ plan,+ ),+ ).toBe(true);+ expect(+ matchesQueryOperators(+ { url: "https://www.news.example/b", title: "Acme Stadium opens" },+ plan,+ ),+ ).toBe(false);+ });+});++describe("siteHostnames", () => {+ it("drops paths and duplicates", () => {+ expect(siteHostnames(["a.com/blog", "a.com", "b.com"])).toEqual([+ "a.com",+ "b.com",+ ]);+ });+});++describe("matchesQueryOperators", () => {+ const financePlan = planOperatorQuery(+ "inurl:earnings Contoso site:finance.example",+ );++ it("enforces site: and the relaxed inurl: on the returned URL", () => {+ expect(+ matchesQueryOperators(+ {+ url: "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",+ },+ financePlan,+ ),+ ).toBe(true);+ expect(+ matchesQueryOperators(+ { url: "https://www.finance.example/quotes/CTSO" },+ financePlan,+ ),+ ).toBe(false);+ expect(+ matchesQueryOperators(+ { url: "https://www.stocks.example/stocks/xnas/ctso/earnings" },+ financePlan,+ ),+ ).toBe(false);+ });++ it("enforces the relaxed intitle: on the title", () => {+ const plan = planOperatorQuery('"Acme" intitle:Acme site:news.example');+ expect(+ matchesQueryOperators(+ {+ url: "https://www.news.example/aktuell/wirtschaft/a.html",+ title: "Acme: Quarterly results - News",+ },+ plan,+ ),+ ).toBe(true);+ expect(+ matchesQueryOperators(+ {+ url: "https://www.news.example/aktuell/sport/b.html",+ title: "Local team wins again - News",+ },+ plan,+ ),+ ).toBe(false);+ });++ it("enforces site: alone, including paths and -site:", () => {+ const plan = planOperatorQuery(+ "Acme site:news.example/aktuell -site:blogs.news.example",+ );+ expect(plan.relaxed).toBe(false);+ expect(+ matchesQueryOperators(+ { url: "https://www.news.example/aktuell/x.html" },+ plan,+ ),+ ).toBe(true);+ expect(+ matchesQueryOperators(+ { url: "https://www.news.example/agenturmeldungen/x.html" },+ plan,+ ),+ ).toBe(false);+ expect(+ matchesQueryOperators(+ { url: "https://blogs.news.example/aktuell/x.html" },+ plan,+ ),+ ).toBe(false);+ expect(+ matchesQueryOperators({ url: "https://forum.example/r/x" }, plan),+ ).toBe(false);+ expect(+ matchesQueryOperators({ url: "https://notnews.example/aktuell/x" }, plan),+ ).toBe(false);+ });++ it("leaves results alone when the query has no site: operator", () => {+ const plan = planOperatorQuery("inurl:earnings Contoso");+ expect(+ matchesQueryOperators(+ { url: "https://quotes.example/quote/CTSO/" },+ plan,+ ),+ ).toBe(true);+ });+});diff --git a/apps/api/src/search/operators.ts b/apps/api/src/search/operators.tsnew file mode 100644index 000000000..7ff2930b6--- /dev/null+++ b/apps/api/src/search/operators.ts@@ -0,0 +1,176 @@+import { domainToASCII } from "node:url";++/**+ * Query operators the search API enforces on returned results, because+ * providers apply site: combined with inurl: or intitle: inconsistently.+ */++export interface QueryOperators {+ sites: string[];+ excludedSites: string[];+ inurl: string[];+ intitle: string[];+ excludedInurl: string[];+ excludedIntitle: string[];+}++export interface OperatorPlan {+ /** The query sent to the provider. */+ query: string;+ operators: QueryOperators;+ /** inurl:/intitle: were stripped from the query and are enforced on results instead. */+ relaxed: boolean;+}++// A quoted phrase matches first, so operator-like text inside it is skipped.+const OPERATOR_TOKEN =+ /("[^"]*")|(^|[\s(])(-?)(site|inurl|intitle):(?:"([^"]*)"|([^\s()]+))(\)*)/gi;++const emptyOperators = (): QueryOperators => ({+ sites: [],+ excludedSites: [],+ inurl: [],+ intitle: [],+ excludedInurl: [],+ excludedIntitle: [],+});++function normalizeSite(value: string): string {+ const site = value.replace(/^[a-z]+:\/\//, "").replace(/\/+$/, "");+ const slash = site.indexOf("/");+ const host = slash === -1 ? site : site.slice(0, slash);+ return (+ (domainToASCII(host) || host) + (slash === -1 ? "" : site.slice(slash))+ );+}++/** True when OR joins anything other than site: operators, which the matcher cannot represent. */+function hasUnsupportedOr(query: string): boolean {+ const unquoted = query.replace(/"[^"]*"/g, " ");+ const ors = unquoted.match(/(?:^|[\s(])(?:OR|\|)(?=[\s)(]|$)/g)?.length ?? 0;+ const siteOrs =+ unquoted.match(/site:[^\s()]+\)*\s+(?:OR|\|)\s+(?=\(*site:)/gi)?.length ??+ 0;+ return ors > siteOrs;+}++export function parseQueryOperators(query: string): QueryOperators {+ const operators = emptyOperators();+ for (const match of query.matchAll(OPERATOR_TOKEN)) {+ if (match[1] !== undefined) continue;+ const value = (match[5] ?? match[6] ?? "").trim().toLowerCase();+ if (!value) continue;+ const negated = match[3] === "-";+ switch (match[4].toLowerCase()) {+ case "site": {+ const site = normalizeSite(value);+ if (site)+ (negated ? operators.excludedSites : operators.sites).push(site);+ break;+ }+ case "inurl":+ (negated ? operators.excludedInurl : operators.inurl).push(value);+ break;+ case "intitle":+ (negated ? operators.excludedIntitle : operators.intitle).push(value);+ break;+ }+ }+ return operators;+}++/**+ * Providers return empty or partly filtered results when site: is combined+ * with inurl: or intitle:, so that combination is sent as site: plus plain+ * terms and the inurl:/intitle: filters are applied to the returned results.+ */+export function planOperatorQuery(query: string): OperatorPlan {+ if (hasUnsupportedOr(query)) {+ return { query, operators: emptyOperators(), relaxed: false };+ }+ const operators = parseQueryOperators(query);+ const relaxed =+ operators.sites.length > 0 &&+ operators.inurl.length ++ operators.intitle.length ++ operators.excludedInurl.length ++ operators.excludedIntitle.length >+ 0;+ if (!relaxed) return { query, operators, relaxed };++ const rewritten = query+ .replace(+ OPERATOR_TOKEN,+ (token, phrase, lead, negated, name, quoted, bare, closing) => {+ if (phrase !== undefined || name.toLowerCase() === "site") return token;+ if (negated) return lead + closing;+ return lead + (quoted !== undefined ? `"${quoted}"` : bare) + closing;+ },+ )+ .replace(/\s+/g, " ")+ .trim();+ return { query: rewritten, operators, relaxed };+}++/** Hostnames from site: operators, for the provider-side domain filter. */+export function siteHostnames(sites: string[]): string[] {+ return [...new Set(sites.map(site => site.split("/")[0]).filter(Boolean))];+}++function matchesSite(url: URL, site: string): boolean {+ const slash = site.indexOf("/");+ const host = slash === -1 ? site : site.slice(0, slash);+ const path = slash === -1 ? "" : site.slice(slash);+ const hostname = url.hostname.toLowerCase();+ if (hostname !== host && !hostname.endsWith("." + host)) return false;+ return !path || url.pathname.toLowerCase().startsWith(path);+}++function decodedUrl(url: string): string {+ try {+ return decodeURI(url).toLowerCase();+ } catch {+ return url.toLowerCase();+ }+}++function inUrl(url: string, term: string): boolean {+ return term.split(/\s+/).every(word => url.includes(word));+}++export function matchesQueryOperators(+ result: { url?: string | null; title?: string | null },+ plan: OperatorPlan,+): boolean {+ const { operators, relaxed } = plan;+ const filtersSites =+ operators.sites.length > 0 || operators.excludedSites.length > 0;+ if (!filtersSites && !relaxed) return true;+ if (!result.url) return false;++ let parsed: URL;+ try {+ parsed = new URL(result.url);+ } catch {+ return false;+ }+ if (operators.excludedSites.some(site => matchesSite(parsed, site))) {+ return false;+ }+ if (+ operators.sites.length > 0 &&+ !operators.sites.some(site => matchesSite(parsed, site))+ ) {+ return false;+ }+ if (!relaxed) return true;++ const url = decodedUrl(result.url);+ const title = (result.title ?? "").toLowerCase();+ return (+ operators.inurl.every(term => inUrl(url, term)) &&+ !operators.excludedInurl.some(term => inUrl(url, term)) &&+ operators.intitle.every(term => title.includes(term)) &&+ !operators.excludedIntitle.some(term => title.includes(term))+ );+}