Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add __experimental_omceDomain flag for debugging and benchmarking (#1844)

devin-ai-integration[bot]a08f343a1302
diff --git a/apps/api/src/__tests__/snips/generateDomainSplits.test.ts b/apps/api/src/__tests__/snips/generateDomainSplits.test.ts
new file mode 100644
index 000000000..915cd86e7
--- /dev/null
+++ b/apps/api/src/__tests__/snips/generateDomainSplits.test.ts
@@ -0,0 +1,43 @@
+import { generateDomainSplits } from "../../services/index";
+
+describe("generateDomainSplits", () => {
+ it("should return original domain splits when no fake domain provided", () => {
+ const result = generateDomainSplits("example.com");
+ expect(result).toEqual(["example.com"]);
+ });
+
+ it("should return fake domain when provided without subdomains", () => {
+ const result = generateDomainSplits("example.com", "fake.com");
+ expect(result).toEqual(["fake.com"]);
+ });
+
+ it("should apply same splitting logic to fake domain with subdomains", () => {
+ const result = generateDomainSplits("sub.example.com", "api.fake.com");
+ expect(result).toEqual(["fake.com", "api.fake.com"]);
+ });
+
+ it("should apply splitting logic to fake domain with multiple subdomains", () => {
+ const result = generateDomainSplits("a.b.example.com", "x.y.fake.org");
+ expect(result).toEqual(["fake.org", "y.fake.org", "x.y.fake.org"]);
+ });
+
+ it("should handle www subdomain in fake domain correctly", () => {
+ const result = generateDomainSplits("www.example.com", "www.fake.com");
+ expect(result).toEqual(["fake.com"]);
+ });
+
+ it("should return fake domain when original hostname is invalid", () => {
+ const result = generateDomainSplits("invalid", "fake.com");
+ expect(result).toEqual(["fake.com"]);
+ });
+
+ it("should return fake domain when fake domain is invalid", () => {
+ const result = generateDomainSplits("example.com", "invalid");
+ expect(result).toEqual(["invalid"]);
+ });
+
+ it("should handle complex subdomain structures with fake domain", () => {
+ const result = generateDomainSplits("api.v1.service.example.com", "sub.test.org");
+ expect(result).toEqual(["test.org", "sub.test.org"]);
+ });
+});
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 6b1b25915..a28b469ca 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -964,4 +964,29 @@ describe("Scrape tests", () => {
expect(response.markdown).toContain("```json");
}, scrapeTimeout);
+
+ describe("__experimental_omceDomain functionality", () => {
+ it.concurrent("should accept __experimental_omceDomain flag in scrape request", async () => {
+ const response = await scrape({
+ url: "https://httpbin.org/html",
+ __experimental_omceDomain: "fake-domain.com",
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.metadata).toBeDefined();
+ }, scrapeTimeout);
+
+ it.concurrent("should work with __experimental_omceDomain and other experimental flags", async () => {
+ const response = await scrape({
+ url: "https://httpbin.org/html",
+ __experimental_omceDomain: "test-domain.org",
+ __experimental_omce: true,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.metadata).toBeDefined();
+ }, scrapeTimeout);
+ });
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 8c6c04f33..51c3e51fe 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -392,6 +392,7 @@ const baseScrapeOptions = z
__experimental_cache: z.boolean().default(false).optional(),
__searchPreviewToken: z.string().optional(),
__experimental_omce: z.boolean().default(false).optional(),
+ __experimental_omceDomain: z.string().optional(),
})
.strict(strictMessage);
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 2e37b2ba7..af67c9e91 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -39,7 +39,8 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
const urlObj = new URL(normalizedURL);
const hostname = urlObj.hostname;
- const domainSplits = generateDomainSplits(hostname);
+ const fakeDomain = meta.options.__experimental_omceDomain;
+ const domainSplits = generateDomainSplits(hostname, fakeDomain);
const domainSplitsHash = domainSplits.map(split => hashURL(split));
const indexId = crypto.randomUUID();
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 3dd372cac..f6e2d3749 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -558,7 +558,8 @@ export async function scrapeURL(
? Date.now() - new Date(data[0].created_at).getTime()
: -1;
- const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname);
+ const fakeDomain = meta.options.__experimental_omceDomain;
+ const domainSplits = generateDomainSplits(new URL(normalizeURLForIndex(meta.url)).hostname, fakeDomain);
const domainHash = hashURL(domainSplits.slice(-1)[0]);
const out = {
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 7d711b906..d0f7d5375 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -188,7 +188,27 @@ export function generateURLSplits(url: string): string[] {
return [...new Set(urls.map(x => normalizeURLForIndex(x)))];
}
-export function generateDomainSplits(hostname: string): string[] {
+export function generateDomainSplits(hostname: string, fakeDomain?: string): string[] {
+ if (fakeDomain) {
+ const parsed = psl.parse(hostname);
+ if (parsed === null) return [fakeDomain];
+
+ const fakeParsed = psl.parse(fakeDomain);
+ if (fakeParsed === null || fakeParsed.domain === null) return [fakeDomain];
+
+ const subdomains: string[] = (fakeParsed.subdomain ?? "").split(".").filter(x => x !== "");
+ if (subdomains.length === 1 && subdomains[0] === "www") {
+ return [fakeParsed.domain];
+ }
+
+ const domains: string[] = [];
+ for (let i = subdomains.length; i >= 0; i--) {
+ domains.push(subdomains.slice(i).concat([fakeParsed.domain]).join("."));
+ }
+
+ return domains;
+ }
+
const parsed = psl.parse(hostname);
if (parsed === null) {
return [];