Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add __experimental_omceDomain flag for debugging and benchmarking (#1844)
devin-ai-integration[bot]a08f343a1302
diff --git a/apps/api/src/__tests__/snips/generateDomainSplits.test.ts b/apps/api/src/__tests__/snips/generateDomainSplits.test.tsnew file mode 100644index 000000000..915cd86e7--- /dev/null+++ b/apps/api/src/__tests__/snips/generateDomainSplits.test.ts@@ -0,0 +1,43 @@+import { generateDomainSplits } from "../../services/index";++describe("generateDomainSplits", () => {+ it("should return original domain splits when no fake domain provided", () => {+ const result = generateDomainSplits("example.com");+ expect(result).toEqual(["example.com"]);+ });++ it("should return fake domain when provided without subdomains", () => {+ const result = generateDomainSplits("example.com", "fake.com");+ expect(result).toEqual(["fake.com"]);+ });++ it("should apply same splitting logic to fake domain with subdomains", () => {+ const result = generateDomainSplits("sub.example.com", "api.fake.com");+ expect(result).toEqual(["fake.com", "api.fake.com"]);+ });++ it("should apply splitting logic to fake domain with multiple subdomains", () => {+ const result = generateDomainSplits("a.b.example.com", "x.y.fake.org");+ expect(result).toEqual(["fake.org", "y.fake.org", "x.y.fake.org"]);+ });++ it("should handle www subdomain in fake domain correctly", () => {+ const result = generateDomainSplits("www.example.com", "www.fake.com");+ expect(result).toEqual(["fake.com"]);+ });++ it("should return fake domain when original hostname is invalid", () => {+ const result = generateDomainSplits("invalid", "fake.com");+ expect(result).toEqual(["fake.com"]);+ });++ it("should return fake domain when fake domain is invalid", () => {+ const result = generateDomainSplits("example.com", "invalid");+ expect(result).toEqual(["invalid"]);+ });++ it("should handle complex subdomain structures with fake domain", () => {+ const result = generateDomainSplits("api.v1.service.example.com", "sub.test.org");+ expect(result).toEqual(["test.org", "sub.test.org"]);+ });+});diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 6b1b25915..a28b469ca 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -964,4 +964,29 @@ describe("Scrape tests", () => {expect(response.markdown).toContain("```json");}, scrapeTimeout);++ describe("__experimental_omceDomain functionality", () => {+ it.concurrent("should accept __experimental_omceDomain flag in scrape request", async () => {+ const response = await scrape({+ url: "https://httpbin.org/html",+ __experimental_omceDomain: "fake-domain.com",+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.metadata).toBeDefined();+ }, scrapeTimeout);++ it.concurrent("should work with __experimental_omceDomain and other experimental flags", async () => {+ const response = await scrape({+ url: "https://httpbin.org/html",+ __experimental_omceDomain: "test-domain.org",+ __experimental_omce: true,+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.metadata).toBeDefined();+ }, scrapeTimeout);+ });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 8c6c04f33..51c3e51fe 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -392,6 +392,7 @@ const baseScrapeOptions = z__experimental_cache: z.boolean().default(false).optional(),__searchPreviewToken: z.string().optional(),__experimental_omce: z.boolean().default(false).optional(),+ __experimental_omceDomain: z.string().optional(),}).strict(strictMessage);diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 2e37b2ba7..af67c9e91 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -39,7 +39,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {const urlObj = new URL(normalizedURL);const hostname = urlObj.hostname;- const domainSplits = generateDomainSplits(hostname);+ const fakeDomain = meta.options.__experimental_omceDomain;+ const domainSplits = generateDomainSplits(hostname, fakeDomain);const domainSplitsHash = domainSplits.map(split => hashURL(split));const indexId = crypto.randomUUID();diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 3dd372cac..f6e2d3749 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -558,7 +558,8 @@ export async function scrapeURL(? Date.now() - new Date(data[0].created_at).getTime(): -1;- const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);+ const fakeDomain = meta.options.__experimental_omceDomain;+ const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname, fakeDomain);const domainHash = hashURL(domainSplits.slice(-1)[0]);const out = {diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 7d711b906..d0f7d5375 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -188,7 +188,27 @@ export function generateURLSplits(url: string): string[] {return [...new Set(urls.map(x => normalizeURLForIndex(x)))];}-export function generateDomainSplits(hostname: string): string[] {+export function generateDomainSplits(hostname: string, fakeDomain?: string): string[] {+ if (fakeDomain) {+ const parsed = psl.parse(hostname);+ if (parsed === null) return [fakeDomain];++ const fakeParsed = psl.parse(fakeDomain);+ if (fakeParsed === null || fakeParsed.domain === null) return [fakeDomain];++ const subdomains: string[] = (fakeParsed.subdomain ?? "").split(".").filter(x => x !== "");+ if (subdomains.length === 1 && subdomains[0] === "www") {+ return [fakeParsed.domain];+ }++ const domains: string[] = [];+ for (let i = subdomains.length; i >= 0; i--) {+ domains.push(subdomains.slice(i).concat([fakeParsed.domain]).join("."));+ }++ return domains;+ }+const parsed = psl.parse(hostname);if (parsed === null) {return [];