Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: Add iframe selector transformation for includeTags and excludeTags (#1850)

devin-ai-integration[bot]659e05b72144
diff --git a/apps/api/src/__tests__/snips/iframe-selectors.test.ts b/apps/api/src/__tests__/snips/iframe-selectors.test.ts
new file mode 100644
index 000000000..1f64a40bf
--- /dev/null
+++ b/apps/api/src/__tests__/snips/iframe-selectors.test.ts
@@ -0,0 +1,73 @@
+import { describe, it, expect } from '@jest/globals';
+import { scrapeOptions } from '../../controllers/v1/types';
+
+describe('iframe selector transformation', () => {
+ describe('Zod schema transformation', () => {
+ it('should transform includeTags through schema', () => {
+ const input = { includeTags: ['iframe', 'iframe.video', 'div.content'] };
+ const result = scrapeOptions.parse(input);
+ expect(result.includeTags).toEqual([
+ 'div[data-original-tag="iframe"]',
+ 'div[data-original-tag="iframe"].video',
+ 'div.content'
+ ]);
+ });
+
+ it('should transform excludeTags through schema', () => {
+ const input = { excludeTags: ['iframe', 'iframe#main-video'] };
+ const result = scrapeOptions.parse(input);
+ expect(result.excludeTags).toEqual([
+ 'div[data-original-tag="iframe"]',
+ 'div[data-original-tag="iframe"]#main-video'
+ ]);
+ });
+
+ it('should handle empty arrays', () => {
+ const input = { includeTags: [], excludeTags: [] };
+ const result = scrapeOptions.parse(input);
+ expect(result.includeTags).toEqual([]);
+ expect(result.excludeTags).toEqual([]);
+ });
+
+ it('should handle missing optional fields', () => {
+ const input = {};
+ const result = scrapeOptions.parse(input);
+ expect(result.includeTags).toBeUndefined();
+ expect(result.excludeTags).toBeUndefined();
+ });
+
+ it('should handle complex selectors with combinators', () => {
+ const input = {
+ includeTags: ['div iframe', 'iframe > div', 'div + iframe', 'iframe ~ p'],
+ excludeTags: ['iframe, iframe.video']
+ };
+ const result = scrapeOptions.parse(input);
+ expect(result.includeTags).toEqual([
+ 'div div[data-original-tag="iframe"]',
+ 'div[data-original-tag="iframe"] > div',
+ 'div + div[data-original-tag="iframe"]',
+ 'div[data-original-tag="iframe"] ~ p'
+ ]);
+ expect(result.excludeTags).toEqual([
+ 'div[data-original-tag="iframe"], div[data-original-tag="iframe"].video'
+ ]);
+ });
+
+ it('should not transform partial matches', () => {
+ const input = {
+ includeTags: ['div.iframe-container', 'iframe-wrapper', 'my-iframe'],
+ excludeTags: ['iframe.embed-responsive-item', '.container iframe[src*="youtube.com"]']
+ };
+ const result = scrapeOptions.parse(input);
+ expect(result.includeTags).toEqual([
+ 'div.iframe-container',
+ 'iframe-wrapper',
+ 'my-iframe'
+ ]);
+ expect(result.excludeTags).toEqual([
+ 'div[data-original-tag="iframe"].embed-responsive-item',
+ '.container div[data-original-tag="iframe"][src*="youtube.com"]'
+ ]);
+ });
+ });
+});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 51c3e51fe..cdf08a345 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -273,6 +273,13 @@ export const actionsSchema = z
},
);
+function transformIframeSelector(selector: string): string {
+ return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => {
+ const prefix = match.match(/^[\s,]/)?.[0] || '';
+ return prefix + 'div[data-original-tag="iframe"]';
+ });
+}
+
const baseScrapeOptions = z
.object({
formats: z
@@ -299,8 +306,12 @@ const baseScrapeOptions = z
"The changeTracking format requires the markdown format to be specified as well",
),
headers: z.record(z.string(), z.string()).optional(),
- includeTags: z.string().array().optional(),
- excludeTags: z.string().array().optional(),
+ includeTags: z.string().array()
+ .transform(tags => tags.map(transformIframeSelector))
+ .optional(),
+ excludeTags: z.string().array()
+ .transform(tags => tags.map(transformIframeSelector))
+ .optional(),
onlyMainContent: z.boolean().default(true),
timeout: z.number().int().positive().finite().safe().optional(),
waitFor: z