Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/search): enforce site:, inurl: and intitle: operator combinations (#5381)

Rakshith Ramprakash13673be17227
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 1785e3412..b3d9dd007 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -616,3 +616,214 @@ describe("executeSearch gov category", () => {
expect(result.searchCredits).toBe(0);
});
});
+
+describe("executeSearch query operators", () => {
+ const web = (url: string, title = "Result") => ({
+ url,
+ title,
+ description: "",
+ });
+ const news = (url: string, title: string) => ({ url, title, snippet: "" });
+
+ it("sends inurl: with site: as plain terms and enforces both on web results", async () => {
+ mocks.search.mockResolvedValue({
+ web: [
+ web(
+ "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",
+ ),
+ web("https://www.finance.example/quotes/CTSO"),
+ web("https://www.stocks.example/stocks/xnas/ctso/earnings"),
+ web(
+ "https://www.finance.example/2026/04/29/contoso-ctso-q3-earnings-report-2026.html",
+ ),
+ ],
+ });
+
+ const result = await executeSearch(
+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },
+ context,
+ logger,
+ );
+
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({
+ query: "earnings Contoso site:finance.example",
+ includeDomains: ["finance.example"],
+ }),
+ );
+ expect(result.response.web?.map(x => [x.position, x.url])).toEqual([
+ [
+ 1,
+ "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",
+ ],
+ [
+ 2,
+ "https://www.finance.example/2026/04/29/contoso-ctso-q3-earnings-report-2026.html",
+ ],
+ ]);
+ });
+
+ it("bills nothing when the operators leave no results", async () => {
+ mocks.search.mockResolvedValue({
+ web: [web("https://www.finance.example/quotes/CTSO")],
+ });
+
+ const result = await executeSearch(
+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },
+ context,
+ logger,
+ );
+
+ expect(result.response.web).toEqual([]);
+ expect(result.totalResultsCount).toBe(0);
+ expect(result.searchCredits).toBe(0);
+ });
+
+ it("applies both intitle: and site: to news results", async () => {
+ mocks.search.mockResolvedValue({
+ news: [
+ news(
+ "https://www.news.example/aktuell/wirtschaft/a.html",
+ "Acme: Quarterly results - News",
+ ),
+ news(
+ "https://www.forum.example/r/insurance/comments/1/acme/",
+ "Acme denied claim",
+ ),
+ news(
+ "https://www.news.example/agenturmeldungen/dpa/b.html",
+ "Local team wins again - News",
+ ),
+ ],
+ });
+
+ const result = await executeSearch(
+ {
+ ...options([]),
+ query: '"Acme" intitle:Acme site:news.example',
+ sources: [{ type: "news" }],
+ tbs: "qdr:w",
+ },
+ context,
+ logger,
+ );
+
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({
+ query: '"Acme" Acme site:news.example',
+ includeDomains: ["news.example"],
+ tbs: "qdr:w",
+ }),
+ );
+ expect(result.response.news?.map(x => x.url)).toEqual([
+ "https://www.news.example/aktuell/wirtschaft/a.html",
+ ]);
+ });
+
+ it("enforces site: alone on news results", async () => {
+ mocks.search.mockResolvedValue({
+ news: [
+ news("https://www.news.example/aktuell/a.html", "A"),
+ news("https://www.acmecare.example/en/blog.html", "B"),
+ ],
+ });
+
+ const result = await executeSearch(
+ {
+ ...options([]),
+ query: "Acme site:news.example",
+ sources: [{ type: "news" }],
+ },
+ context,
+ logger,
+ );
+
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ query: "Acme site:news.example" }),
+ );
+ expect(result.response.news?.map(x => x.url)).toEqual([
+ "https://www.news.example/aktuell/a.html",
+ ]);
+ });
+
+ it("enforces site: on image results", async () => {
+ mocks.search.mockResolvedValue({
+ images: [
+ {
+ url: "https://www.news.example/a.html",
+ title: "A",
+ imageUrl: "https://cdn.example/a.jpg",
+ },
+ {
+ url: "https://other.example/b.html",
+ title: "B",
+ imageUrl: "https://cdn.example/b.jpg",
+ },
+ ],
+ });
+
+ const result = await executeSearch(
+ {
+ ...options([]),
+ query: "Acme site:news.example",
+ sources: [{ type: "images" }],
+ },
+ context,
+ logger,
+ );
+
+ expect(result.response.images?.map(x => [x.position, x.url])).toEqual([
+ [1, "https://www.news.example/a.html"],
+ ]);
+ });
+
+ it("enforces site: on category index results", async () => {
+ mocks.searchDeveloperCategory.mockResolvedValue([
+ developerResult,
+ { ...developerResult, url: "https://docs.example/retries", position: 2 },
+ ]);
+
+ const result = await executeSearch(
+ {
+ ...options([{ type: "developer" }]),
+ query: "retries site:docs.example",
+ },
+ context,
+ logger,
+ );
+
+ expect(result.response.web?.map(x => [x.position, x.url])).toEqual([
+ [1, "https://docs.example/retries"],
+ ]);
+ expect(result.indexResultsCount).toBe(1);
+ });
+
+ it("asks the provider for more results when it enforces inurl: or intitle: itself", async () => {
+ mocks.search.mockResolvedValue({ web: [] });
+
+ await executeSearch(
+ { ...options([]), query: "inurl:earnings Contoso site:finance.example" },
+ context,
+ logger,
+ );
+
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ num_results: 50 }),
+ );
+ });
+
+ it("forwards a custom date range tbs to the web provider unchanged", async () => {
+ mocks.search.mockResolvedValue({ web: [] });
+ const tbs = "cdr:1,cd_min:1/1/2024,cd_max:1/31/2024";
+
+ await executeSearch(
+ { ...options([]), query: "Contoso earnings site:finance.example", tbs },
+ context,
+ logger,
+ );
+
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ tbs, type: ["web"] }),
+ );
+ });
+});
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 484f535f9..40548fd3e 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -29,6 +29,11 @@ import {
import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";
import { searchGovCategory, wantsGovCategory } from "./gov";
import { removeExplicitResults } from "./safe-search";
+import {
+ matchesQueryOperators,
+ planOperatorQuery,
+ siteHostnames,
+} from "./operators";
import {
highlightsEnvReady,
runIndexedSearchHighlights,
@@ -190,8 +195,11 @@ export async function executeSearch(
.map(source => source.type as SearchResultType),
),
];
+ const operatorPlan = planOperatorQuery(query);
+ const { operators } = operatorPlan;
+ const nonEmpty = (list: string[]) => (list.length > 0 ? list : undefined);
const { query: searchQuery, categoryMap } = buildSearchQuery(
- query,
+ operatorPlan.query,
categories,
{
includeDomains: options.includeDomains,
@@ -231,7 +239,9 @@ export async function executeSearch(
logger,
requestId: context.requestId,
advanced: false,
- num_results: num_results_buffer,
+ num_results: operatorPlan.relaxed
+ ? Math.max(num_results_buffer, Math.min(limit * 5, 100))
+ : num_results_buffer,
tbs: options.tbs,
filter: options.filter,
lang: options.lang,
@@ -240,10 +250,33 @@ export async function executeSearch(
safe: options.safe,
type: searchTypes,
enterprise: options.enterprise,
- includeDomains: options.includeDomains,
- excludeDomains: options.excludeDomains,
+ includeDomains: options.includeDomains?.length
+ ? options.includeDomains
+ : nonEmpty(siteHostnames(operators.sites)),
+ excludeDomains: nonEmpty([
+ ...(options.excludeDomains ?? []),
+ ...operators.excludedSites.filter(site => !site.includes("/")),
+ ]),
})) as SearchV2Response);
- let indexResults = indexResultsPromise ? await indexResultsPromise : [];
+ const keepOperatorMatches = <
+ T extends { url?: string; title?: string; position?: number },
+ >(
+ results: T[] | undefined,
+ ): T[] | undefined => {
+ const kept = results?.filter(result =>
+ matchesQueryOperators(result, operatorPlan),
+ );
+ return kept?.length === results?.length
+ ? results
+ : kept?.map((result, index) => ({ ...result, position: index + 1 }));
+ };
+ searchResponse.web = keepOperatorMatches(searchResponse.web);
+ searchResponse.news = keepOperatorMatches(searchResponse.news);
+ searchResponse.images = keepOperatorMatches(searchResponse.images);
+ let indexResults =
+ keepOperatorMatches(
+ indexResultsPromise ? await indexResultsPromise : undefined,
+ ) ?? [];
// Threat protection: remove blocked results entirely — before
// slicing/counting, before scraping, and before returning. Checks are
diff --git a/apps/api/src/search/operators.test.ts b/apps/api/src/search/operators.test.ts
new file mode 100644
index 000000000..29c5d1056
--- /dev/null
+++ b/apps/api/src/search/operators.test.ts
@@ -0,0 +1,218 @@
+import { describe, expect, it } from "vitest";
+import {
+ matchesQueryOperators,
+ parseQueryOperators,
+ planOperatorQuery,
+ siteHostnames,
+} from "./operators";
+
+describe("parseQueryOperators", () => {
+ it("reads site:, -site:, inurl:, intitle: and quoted values", () => {
+ expect(
+ parseQueryOperators(
+ '"Acme" intitle:"Acme Corp" inurl:earnings -inurl:video site:news.example -site:jobs.news.example',
+ ),
+ ).toEqual({
+ sites: ["news.example"],
+ excludedSites: ["jobs.news.example"],
+ inurl: ["earnings"],
+ intitle: ["acme corp"],
+ excludedInurl: ["video"],
+ excludedIntitle: [],
+ });
+ });
+
+ it("reads site: inside an OR group and ignores allintitle:", () => {
+ const operators = parseQueryOperators(
+ "(site:a.com OR site:https://b.com/docs/) allintitle:foo",
+ );
+ expect(operators.sites).toEqual(["a.com", "b.com/docs"]);
+ expect(operators.intitle).toEqual([]);
+ });
+});
+
+describe("planOperatorQuery", () => {
+ it("ignores operator-like text inside quoted phrases", () => {
+ const plan = planOperatorQuery('"Acme inurl:earnings" site:news.example');
+ expect(plan.operators.inurl).toEqual([]);
+ expect(plan.relaxed).toBe(false);
+ expect(plan.query).toBe('"Acme inurl:earnings" site:news.example');
+ });
+
+ it("enforces nothing when OR joins anything but site: operators", () => {
+ for (const query of [
+ "(site:news.example OR intitle:Acme)",
+ "Acme site:news.example OR earnings",
+ ]) {
+ const plan = planOperatorQuery(query);
+ expect(plan.query).toBe(query);
+ expect(plan.relaxed).toBe(false);
+ expect(plan.operators.sites).toEqual([]);
+ expect(
+ matchesQueryOperators({ url: "https://other.example/" }, plan),
+ ).toBe(true);
+ }
+ });
+
+ it("keeps an OR chain of site: operators", () => {
+ expect(
+ planOperatorQuery(
+ "Acme (site:a.example OR site:b.example OR site:c.example)",
+ ).operators.sites,
+ ).toEqual(["a.example", "b.example", "c.example"]);
+ });
+
+ it("keeps the closing parenthesis of a dropped negated operator", () => {
+ expect(
+ planOperatorQuery("site:news.example (Acme -inurl:video)").query,
+ ).toBe("site:news.example (Acme )");
+ });
+
+ it("normalizes unicode site: hosts to ASCII", () => {
+ const plan = planOperatorQuery("Acme site:bücher.example");
+ expect(plan.operators.sites).toEqual(["xn--bcher-kva.example"]);
+ expect(
+ matchesQueryOperators({ url: "https://www.bücher.example/a" }, plan),
+ ).toBe(true);
+ });
+
+ it("keeps a query without the site: plus inurl:/intitle: combination", () => {
+ for (const query of [
+ "earnings Contoso site:finance.example",
+ "inurl:earnings Contoso",
+ "intitle:Acme",
+ ]) {
+ const plan = planOperatorQuery(query);
+ expect(plan.query).toBe(query);
+ expect(plan.relaxed).toBe(false);
+ }
+ });
+
+ it("sends inurl: with site: as a plain term", () => {
+ const plan = planOperatorQuery(
+ "inurl:earnings Contoso site:finance.example",
+ );
+ expect(plan.relaxed).toBe(true);
+ expect(plan.query).toBe("earnings Contoso site:finance.example");
+ });
+
+ it("sends intitle: with site: as a plain term and enforces negated filters on results", () => {
+ const plan = planOperatorQuery(
+ '"Acme" intitle:Acme -intitle:Stadium site:news.example',
+ );
+ expect(plan.query).toBe('"Acme" Acme site:news.example');
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.news.example/a", title: "Acme results" },
+ plan,
+ ),
+ ).toBe(true);
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.news.example/b", title: "Acme Stadium opens" },
+ plan,
+ ),
+ ).toBe(false);
+ });
+});
+
+describe("siteHostnames", () => {
+ it("drops paths and duplicates", () => {
+ expect(siteHostnames(["a.com/blog", "a.com", "b.com"])).toEqual([
+ "a.com",
+ "b.com",
+ ]);
+ });
+});
+
+describe("matchesQueryOperators", () => {
+ const financePlan = planOperatorQuery(
+ "inurl:earnings Contoso site:finance.example",
+ );
+
+ it("enforces site: and the relaxed inurl: on the returned URL", () => {
+ expect(
+ matchesQueryOperators(
+ {
+ url: "https://www.finance.example/2026/07/29/contoso-ctso-q4-earnings-report-2026.html",
+ },
+ financePlan,
+ ),
+ ).toBe(true);
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.finance.example/quotes/CTSO" },
+ financePlan,
+ ),
+ ).toBe(false);
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.stocks.example/stocks/xnas/ctso/earnings" },
+ financePlan,
+ ),
+ ).toBe(false);
+ });
+
+ it("enforces the relaxed intitle: on the title", () => {
+ const plan = planOperatorQuery('"Acme" intitle:Acme site:news.example');
+ expect(
+ matchesQueryOperators(
+ {
+ url: "https://www.news.example/aktuell/wirtschaft/a.html",
+ title: "Acme: Quarterly results - News",
+ },
+ plan,
+ ),
+ ).toBe(true);
+ expect(
+ matchesQueryOperators(
+ {
+ url: "https://www.news.example/aktuell/sport/b.html",
+ title: "Local team wins again - News",
+ },
+ plan,
+ ),
+ ).toBe(false);
+ });
+
+ it("enforces site: alone, including paths and -site:", () => {
+ const plan = planOperatorQuery(
+ "Acme site:news.example/aktuell -site:blogs.news.example",
+ );
+ expect(plan.relaxed).toBe(false);
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.news.example/aktuell/x.html" },
+ plan,
+ ),
+ ).toBe(true);
+ expect(
+ matchesQueryOperators(
+ { url: "https://www.news.example/agenturmeldungen/x.html" },
+ plan,
+ ),
+ ).toBe(false);
+ expect(
+ matchesQueryOperators(
+ { url: "https://blogs.news.example/aktuell/x.html" },
+ plan,
+ ),
+ ).toBe(false);
+ expect(
+ matchesQueryOperators({ url: "https://forum.example/r/x" }, plan),
+ ).toBe(false);
+ expect(
+ matchesQueryOperators({ url: "https://notnews.example/aktuell/x" }, plan),
+ ).toBe(false);
+ });
+
+ it("leaves results alone when the query has no site: operator", () => {
+ const plan = planOperatorQuery("inurl:earnings Contoso");
+ expect(
+ matchesQueryOperators(
+ { url: "https://quotes.example/quote/CTSO/" },
+ plan,
+ ),
+ ).toBe(true);
+ });
+});
diff --git a/apps/api/src/search/operators.ts b/apps/api/src/search/operators.ts
new file mode 100644
index 000000000..7ff2930b6
--- /dev/null
+++ b/apps/api/src/search/operators.ts
@@ -0,0 +1,176 @@
+import { domainToASCII } from "node:url";
+
+/**
+ * Query operators the search API enforces on returned results, because
+ * providers apply site: combined with inurl: or intitle: inconsistently.
+ */
+
+export interface QueryOperators {
+ sites: string[];
+ excludedSites: string[];
+ inurl: string[];
+ intitle: string[];
+ excludedInurl: string[];
+ excludedIntitle: string[];
+}
+
+export interface OperatorPlan {
+ /** The query sent to the provider. */
+ query: string;
+ operators: QueryOperators;
+ /** inurl:/intitle: were stripped from the query and are enforced on results instead. */
+ relaxed: boolean;
+}
+
+// A quoted phrase matches first, so operator-like text inside it is skipped.
+const OPERATOR_TOKEN =
+ /("[^"]*")|(^|[\s(])(-?)(site|inurl|intitle):(?:"([^"]*)"|([^\s()]+))(\)*)/gi;
+
+const emptyOperators = (): QueryOperators => ({
+ sites: [],
+ excludedSites: [],
+ inurl: [],
+ intitle: [],
+ excludedInurl: [],
+ excludedIntitle: [],
+});
+
+function normalizeSite(value: string): string {
+ const site = value.replace(/^[a-z]+:\/\//, "").replace(/\/+$/, "");
+ const slash = site.indexOf("/");
+ const host = slash === -1 ? site : site.slice(0, slash);
+ return (
+ (domainToASCII(host) || host) + (slash === -1 ? "" : site.slice(slash))
+ );
+}
+
+/** True when OR joins anything other than site: operators, which the matcher cannot represent. */
+function hasUnsupportedOr(query: string): boolean {
+ const unquoted = query.replace(/"[^"]*"/g, " ");
+ const ors = unquoted.match(/(?:^|[\s(])(?:OR|\|)(?=[\s)(]|$)/g)?.length ?? 0;
+ const siteOrs =
+ unquoted.match(/site:[^\s()]+\)*\s+(?:OR|\|)\s+(?=\(*site:)/gi)?.length ??
+ 0;
+ return ors > siteOrs;
+}
+
+export function parseQueryOperators(query: string): QueryOperators {
+ const operators = emptyOperators();
+ for (const match of query.matchAll(OPERATOR_TOKEN)) {
+ if (match[1] !== undefined) continue;
+ const value = (match[5] ?? match[6] ?? "").trim().toLowerCase();
+ if (!value) continue;
+ const negated = match[3] === "-";
+ switch (match[4].toLowerCase()) {
+ case "site": {
+ const site = normalizeSite(value);
+ if (site)
+ (negated ? operators.excludedSites : operators.sites).push(site);
+ break;
+ }
+ case "inurl":
+ (negated ? operators.excludedInurl : operators.inurl).push(value);
+ break;
+ case "intitle":
+ (negated ? operators.excludedIntitle : operators.intitle).push(value);
+ break;
+ }
+ }
+ return operators;
+}
+
+/**
+ * Providers return empty or partly filtered results when site: is combined
+ * with inurl: or intitle:, so that combination is sent as site: plus plain
+ * terms and the inurl:/intitle: filters are applied to the returned results.
+ */
+export function planOperatorQuery(query: string): OperatorPlan {
+ if (hasUnsupportedOr(query)) {
+ return { query, operators: emptyOperators(), relaxed: false };
+ }
+ const operators = parseQueryOperators(query);
+ const relaxed =
+ operators.sites.length > 0 &&
+ operators.inurl.length +
+ operators.intitle.length +
+ operators.excludedInurl.length +
+ operators.excludedIntitle.length >
+ 0;
+ if (!relaxed) return { query, operators, relaxed };
+
+ const rewritten = query
+ .replace(
+ OPERATOR_TOKEN,
+ (token, phrase, lead, negated, name, quoted, bare, closing) => {
+ if (phrase !== undefined || name.toLowerCase() === "site") return token;
+ if (negated) return lead + closing;
+ return lead + (quoted !== undefined ? `"${quoted}"` : bare) + closing;
+ },
+ )
+ .replace(/\s+/g, " ")
+ .trim();
+ return { query: rewritten, operators, relaxed };
+}
+
+/** Hostnames from site: operators, for the provider-side domain filter. */
+export function siteHostnames(sites: string[]): string[] {
+ return [...new Set(sites.map(site => site.split("/")[0]).filter(Boolean))];
+}
+
+function matchesSite(url: URL, site: string): boolean {
+ const slash = site.indexOf("/");
+ const host = slash === -1 ? site : site.slice(0, slash);
+ const path = slash === -1 ? "" : site.slice(slash);
+ const hostname = url.hostname.toLowerCase();
+ if (hostname !== host && !hostname.endsWith("." + host)) return false;
+ return !path || url.pathname.toLowerCase().startsWith(path);
+}
+
+function decodedUrl(url: string): string {
+ try {
+ return decodeURI(url).toLowerCase();
+ } catch {
+ return url.toLowerCase();
+ }
+}
+
+function inUrl(url: string, term: string): boolean {
+ return term.split(/\s+/).every(word => url.includes(word));
+}
+
+export function matchesQueryOperators(
+ result: { url?: string | null; title?: string | null },
+ plan: OperatorPlan,
+): boolean {
+ const { operators, relaxed } = plan;
+ const filtersSites =
+ operators.sites.length > 0 || operators.excludedSites.length > 0;
+ if (!filtersSites && !relaxed) return true;
+ if (!result.url) return false;
+
+ let parsed: URL;
+ try {
+ parsed = new URL(result.url);
+ } catch {
+ return false;
+ }
+ if (operators.excludedSites.some(site => matchesSite(parsed, site))) {
+ return false;
+ }
+ if (
+ operators.sites.length > 0 &&
+ !operators.sites.some(site => matchesSite(parsed, site))
+ ) {
+ return false;
+ }
+ if (!relaxed) return true;
+
+ const url = decodedUrl(result.url);
+ const title = (result.title ?? "").toLowerCase();
+ return (
+ operators.inurl.every(term => inUrl(url, term)) &&
+ !operators.excludedInurl.some(term => inUrl(url, term)) &&
+ operators.intitle.every(term => title.includes(term)) &&
+ !operators.excludedIntitle.some(term => title.includes(term))
+ );
+}