Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge remote-tracking branch 'origin/main' into devin/ENG-3639-1759244281
AdemÃlson Tonatoc57ad2d2d3ef
diff --cc apps/api/src/__tests__/snips/v2/scrape-viewport.test.tsindex b2f7c25ba,c60b12f4c..2a19c0368--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts@@@ -1,189 -1,181 +1,184 @@@+ import { describeIf, TEST_PRODUCTION } from "../lib";import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";- describe("V2 Scrape Screenshot Viewport", () => {- if (!process.env.TEST_SUITE_SELF_HOSTED) {- let identity: Identity;-- beforeAll(async () => {- identity = await idmux({- name: "v2-scrape-viewport",- concurrency: 100,- credits: 1000000,- });- }, 10000);-- test(- "should take screenshot with custom viewport dimensions",- async () => {- const data = await scrape(- {- url: "https://example.com",- formats: [- "markdown",- {- type: "screenshot",- viewport: {- width: 1920,- height: 1080,- },- },- ],- },- identity,- );-- expect(data).toBeDefined();- expect(data.screenshot).toBeDefined();- expect(data.markdown).toBeDefined();- },- scrapeTimeout,- );-- test(- "should reject fullPage=true with viewport",- async () => {- const response = await scrapeRaw(- {- url: "https://example.com",- formats: [- {- type: "screenshot",- fullPage: true,- viewport: {- width: 1440,- height: 900,- },+ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "v2-scrape-viewport",+ concurrency: 100,+ credits: 1000000,+ });+ }, 10000);++ test(+ "should take screenshot with custom viewport dimensions",+ async () => {+ const data = await scrape(+ {+ url: "https://example.com",+ formats: [+ "markdown",+ {+ type: "screenshot",+ viewport: {+ width: 1920,+ height: 1080,},- ],- },- identity,- );-- expect(response.status).toBe(400);- expect(response.body.success).toBe(false);- expect(response.body.error).toContain(- "Cannot specify viewport dimensions when fullPage is true",- );- },- scrapeTimeout,- );-- test(- "should work with screenshot format without viewport (backwards compatibility)",- async () => {- const data = await scrape(- {- url: "https://example.com",- formats: ["screenshot"],- },- identity,- );-- expect(data).toBeDefined();- expect(data.screenshot).toBeDefined();- },- scrapeTimeout,- );-- test(- "should work with object screenshot format without viewport",- async () => {- const data = await scrape(- {- url: "https://example.com",- formats: [- {- type: "screenshot",- fullPage: false,+ },+ ],+ },+ identity,+ );++ expect(data).toBeDefined();+ expect(data.screenshot).toBeDefined();+ expect(data.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ test(- "should take full page screenshot with custom viewport width",++ "should reject fullPage=true with viewport",+ async () => {- const data = await scrape(++ const data = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "screenshot",+ fullPage: true,+ viewport: {+ width: 1440,+ height: 900,},- ],- },- identity,- );-- expect(data).toBeDefined();- expect(data.screenshot).toBeDefined();- },- scrapeTimeout,- );-- test(- "should reject invalid viewport dimensions",- async () => {- const response = await scrapeRaw(- {- url: "https://example.com",- formats: [- {- type: "screenshot",- viewport: {- width: -100,- height: 0,- },+ },+ ],+ },+ identity,+ );+- expect(data).toBeDefined();- expect(data.screenshot).toBeDefined();++ expect(response.status).toBe(400);++ expect(response.body.success).toBe(false);++ expect(response.body.error).toContain(++ "Cannot specify viewport dimensions when fullPage is true",++ );+ },+ scrapeTimeout,+ );++ test(+ "should work with screenshot format without viewport (backwards compatibility)",+ async () => {+ const data = await scrape(+ {+ url: "https://example.com",+ formats: ["screenshot"],+ },+ identity,+ );++ expect(data).toBeDefined();+ expect(data.screenshot).toBeDefined();+ },+ scrapeTimeout,+ );++ test(+ "should work with object screenshot format without viewport",+ async () => {+ const data = await scrape(+ {+ url: "https://example.com",+ formats: [+ {+ type: "screenshot",+ fullPage: false,+ },+ ],+ },+ identity,+ );++ expect(data).toBeDefined();+ expect(data.screenshot).toBeDefined();+ },+ scrapeTimeout,+ );++ test(+ "should reject invalid viewport dimensions",+ async () => {+ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "screenshot",+ viewport: {+ width: -100,+ height: 0,},- ],- },- identity,- );-- expect(response.status).toBe(400);- expect(response.body.success).toBe(false);- },- scrapeTimeout,- );-- test(- "should reject non-integer viewport dimensions",- async () => {- const response = await scrapeRaw(- {- url: "https://example.com",- formats: [- {- type: "screenshot",- viewport: {- width: "1920" as any,- height: 1080.5 as any,- },+ },+ ],+ },+ identity,+ );++ expect(response.status).toBe(400);+ expect(response.body.success).toBe(false);+ },+ scrapeTimeout,+ );++ test(+ "should reject non-integer viewport dimensions",+ async () => {+ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "screenshot",+ viewport: {+ width: "1920" as any,+ height: 1080.5 as any,},- ],- },- identity,- );-- expect(response.status).toBe(400);- expect(response.body.success).toBe(false);- },- scrapeTimeout,- );-- test(- "should reject viewport dimensions exceeding maximum limits",- async () => {- const response = await scrapeRaw(- {- url: "https://example.com",- formats: [- {- type: "screenshot",- viewport: {- width: 8000, // exceeds max of 7680- height: 5000, // exceeds max of 4320- },+ },+ ],+ },+ identity,+ );++ expect(response.status).toBe(400);+ expect(response.body.success).toBe(false);+ },+ scrapeTimeout,+ );++ test(+ "should reject viewport dimensions exceeding maximum limits",+ async () => {+ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "screenshot",+ viewport: {+ width: 8000, // exceeds max of 7680+ height: 5000, // exceeds max of 4320},- ],- },- identity,- );-- expect(response.status).toBe(400);- expect(response.body.success).toBe(false);- },- scrapeTimeout,- );- } else {- it("mocked", () => {- expect(true).toBe(true);- });- }+ },+ ],+ },+ identity,+ );++ expect(response.status).toBe(400);+ expect(response.body.success).toBe(false);+ },+ scrapeTimeout,+ );});diff --cc apps/api/src/controllers/v2/f-search.tsindex 000000000,67b56a2ee..80bc74b25mode 000000,100644..100644--- a/apps/api/src/controllers/v2/f-search.ts+++ b/apps/api/src/controllers/v2/f-search.ts@@@ -1,0 -1,230 +1,233 @@@+ import { Request, Response } from "express";+ import { z } from "zod";+ import { logger as _logger } from "../../lib/logger";+ import { CostTracking } from "../../lib/cost-tracking";+ import {+ getSearchIndexClient,+ SearchIndexClient,+ type SearchRequest,+ } from "../../lib/search-index-client";++ // Validation schemas+ const searchRequestSchema = z.object({+ query: z.string().min(1).max(500),+ limit: z.number().int().min(1).max(100).optional().default(50),+ offset: z.number().int().min(0).optional().default(0),- mode: z.enum(["hybrid", "keyword", "semantic", "bm25"]).optional().default("hybrid"),++ mode: z++ .enum(["hybrid", "keyword", "semantic", "bm25"])++ .optional()++ .default("hybrid"),+ filters: z+ .object({+ domain: z.string().optional(),+ country: z.string().length(2).optional(),+ isMobile: z.boolean().optional(),+ minFreshness: z.number().min(0).max(1).optional(),+ language: z.string().optional(),+ })+ .optional()+ .default({}),+ });++ const searchChunksRequestSchema = z.object({+ query: z.string().min(1).max(500),+ limit: z.number().int().min(1).max(50).optional().default(20),+ filters: z+ .object({+ domain: z.string().optional(),+ })+ .optional()+ .default({}),+ });++ export async function realtimeSearchController(+ req: Request,+ res: Response,+ ): Promise<void> {+ const logger = _logger.child({+ module: "realtime-search-controller",+ method: "POST /admin/search",+ teamId: (req as any).auth?.team_id,+ });++ try {+ // Validate request+ const validationResult = searchRequestSchema.safeParse(req.body);++ if (!validationResult.success) {+ res.status(400).json({+ success: false,+ error: "Invalid request parameters",+ details: validationResult.error.errors,+ });+ return;+ }++ const { query, limit, offset, mode, filters } = validationResult.data;++ logger.info("Search request", {+ query,+ mode,+ limit,+ filters,+ });++ // Get search index client+ const client = getSearchIndexClient();-+++ if (!client) {+ res.status(503).json({+ success: false,+ error: "Search index service is not configured",+ });+ return;+ }++ // Perform search via HTTP client+ const searchRequest: SearchRequest = {+ query,+ limit,+ offset,+ mode,+ filters,+ };++ const result = await client.search(searchRequest, logger);++ // Track cost (if applicable)+ const costTracking = new CostTracking();+ // Add search cost tracking here if needed++ res.status(200).json({+ success: true,+ data: result,+ costTracking: costTracking.toJSON(),+ });+ } catch (error) {+ logger.error("Search request failed", {+ error: (error as Error).message,+ });++ res.status(500).json({+ success: false,+ error: "Internal server error",+ message: (error as Error).message,+ });+ }+ }++ // /**+ // * POST /admin/search/chunks+ // * Chunk-level search for precise snippets+ // */+ // export async function searchChunksController(+ // req: Request,+ // res: Response,+ // ): Promise<void> {+ // const logger = _logger.child({+ // module: "search-chunks-controller",+ // method: "POST /admin/search/chunks",+ // teamId: (req as any).auth?.team_id,+ // });++ // try {+ // // Validate request+ // const validationResult = searchChunksRequestSchema.safeParse(req.body);++ // if (!validationResult.success) {+ // res.status(400).json({+ // success: false,+ // error: "Invalid request parameters",+ // details: validationResult.error.errors,+ // });+ // return;+ // }++ // const { query, limit, filters } = validationResult.data;++ // logger.info("Chunk search request", {+ // query,+ // limit,+ // filters,+ // });++ // // Check if search index is enabled+ // if (!isSearchIndexEnabled()) {+ // res.status(503).json({+ // success: false,+ // error: "Search index is not configured",+ // });+ // return;+ // }++ // // Perform chunk search+ // const results = await searchChunks(+ // search_index_supabase_service,+ // query,+ // limit,+ // filters as SearchFilters,+ // logger,+ // );++ // res.status(200).json({+ // success: true,+ // data: {+ // chunks: results,+ // total: results.length,+ // query,+ // },+ // });+ // } catch (error) {+ // logger.error("Chunk search request failed", {+ // error: (error as Error).message,+ // });++ // res.status(500).json({+ // success: false,+ // error: "Internal server error",+ // message: (error as Error).message,+ // });+ // }+ // }++ // /**+ // * GET /admin/search/stats+ // * Get search index statistics+ // */+ // export async function searchStatsController(+ // req: Request,+ // res: Response,+ // ): Promise<void> {+ // const logger = _logger.child({+ // module: "search-stats-controller",+ // method: "GET /admin/search/stats",+ // });++ // try {+ // // Check if search index is enabled+ // if (!isSearchIndexEnabled()) {+ // res.status(503).json({+ // success: false,+ // error: "Search index is not configured",+ // });+ // return;+ // }++ // const stats = await getSearchStats(search_index_supabase_service);++ // res.status(200).json({+ // success: true,+ // data: stats,+ // });+ // } catch (error) {+ // logger.error("Failed to get search stats", {+ // error: (error as Error).message,+ // });++ // res.status(500).json({+ // success: false,+ // error: "Internal server error",+ // message: (error as Error).message,+ // });+ // }+ // }diff --cc apps/api/src/lib/crawl-redis.tsindex e0b777d8a,ff4ee837f..1b256ea67--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@@ -49,6 -49,24 +49,18 @@@ export async function saveCrawl(id: str});}-export async function recordRobotsBlocked(- crawlId: string,- url: string,-) {- await redisEvictConnection.sadd(- "crawl:" + crawlId + ":robots_blocked",- url,- );++export async function recordRobotsBlocked(crawlId: string, url: string) {++ await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);+ await redisEvictConnection.expire(+ "crawl:" + crawlId + ":robots_blocked",+ 24 * 60 * 60,+ );+ }++ export async function markCrawlActive(id: string) {+ await redisEvictConnection.sadd("active_crawls", id);+ }+export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);}diff --cc apps/api/src/lib/search-query-builder.tsindex 613d0fd86,9619d1ec8..ecfc43cf4--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@@ -116,6 -134,8 +134,8 @@@ export function getCategoryFromUrl// Check against category map for other sitesfor (const [site, category] of categoryMap.entries()) {+ if (site === "__pdf__") continue; // Skip the special PDF marker-++if (hostname === site.toLowerCase() ||hostname.endsWith("." + site.toLowerCase())diff --cc apps/api/src/scraper/scrapeURL/engines/document/index.tsindex 649b520c0,8a8cfc8f4..f56d23fb8--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@@ -41,30 -65,94 +65,98 @@@ function getDocumentTypeFromContentTypereturn null;}+ function isValidDocumentContentType(contentType: string | null): boolean {+ if (!contentType) return false;++ const ct = contentType.toLowerCase();+ const validTypes = [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ "application/vnd.ms-excel",+ "application/msword",+ "application/rtf",+ "text/rtf",+ "application/vnd.oasis.opendocument.text",+ ];++ return validTypes.some(type => ct.includes(type));+ }+export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {- const { response, buffer } = await fetchFileToBuffer(- meta.rewrittenUrl ?? meta.url,- {+ let response: Response;+ let buffer: Buffer;+ let proxyUsed: "basic" | "stealth" = "basic";+ let tempFilePath: string | null = null;++ if (meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {+ // Use prefetched document+ tempFilePath = meta.documentPrefetch.filePath;+ buffer = await readFile(tempFilePath);++ // Create a mock response object with content-type from prefetch+ const headers = new Headers();+ if (meta.documentPrefetch.contentType) {+ headers.set("Content-Type", meta.documentPrefetch.contentType);+ }++ response = {+ url: meta.documentPrefetch.url ?? meta.rewrittenUrl ?? meta.url,+ status: meta.documentPrefetch.status,+ headers,+ } as Response;++ proxyUsed = meta.documentPrefetch.proxyUsed;+ } else {+ // Fetch the document normally+ const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,signal: meta.abort.asSignal(),- },- );-- const documentType =- getDocumentTypeFromContentType(response.headers.get("content-type")) ??- getDocumentTypeFromUrl(response.url);-- const html = await converter.convertBufferToHtml(- new Uint8Array(buffer),- documentType,- );-- return {- url: response.url,- statusCode: response.status,- html,- proxyUsed: "basic",- };+ });+ response = result.response;+ buffer = result.buffer;++ // Validate content type only when fetching directly (not using prefetch)+ const ct = response.headers.get("Content-Type");+ if (ct && !isValidDocumentContentType(ct)) {+ // if downloaded file wasn't a valid document, throw antibot error+ throw new DocumentAntibotError();+ }+ }++ try {+ const documentType =+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??+ getDocumentTypeFromUrl(response.url);++ const html = await converter.convertBufferToHtml(+ new Uint8Array(buffer),+ documentType,+ );++ return {+ url: response.url,+ statusCode: response.status,+ html,+ proxyUsed,+ };+ } finally {+ // Clean up temporary file if it was created by prefetch- if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {++ if (++ tempFilePath &&++ meta.documentPrefetch !== undefined &&++ meta.documentPrefetch !== null++ ) {+ try {+ await unlink(tempFilePath);+ } catch (error) {+ // Ignore errors when cleaning up temp files- meta.logger?.warn("Failed to clean up temporary document file", {- error,- tempFilePath++ meta.logger?.warn("Failed to clean up temporary document file", {++ error,++ tempFilePath,+ });+ }+ }+ }}export function documentMaxReasonableTime(meta: Meta): number {diff --cc apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 0f6442e53,7b300fac4..00e7c713e--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@@ -67,6 -67,56 +67,53 @@@ async function scrapePDFWithRunPodMUmaxPages,});--+ if (+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&+ process.env.PDF_MU_V2_BASE_URL &&+ Math.random() * 100 <+ Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")+ ) {+ (async () => {+ const startedAt = Date.now();+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });+ try {+ const resp = await robustFetch({+ url: process.env.PDF_MU_V2_BASE_URL ?? "",+ method: "POST",+ body: {+ input: {+ file_content: base64Content,+ filename: path.basename(tempFilePath) + ".pdf",+ timeout: meta.abort.scrapeTimeout(),+ created_at: Date.now(),+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ },+ logger,+ schema: z.any(),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ const body: any = resp as any;+ const tokensIn = body?.metadata?.["total-input-tokens"];+ const tokensOut = body?.metadata?.["total-output-tokens"];+ const pages = body?.metadata?.["pdf-total-pages"];+ const durationMs = Date.now() - startedAt;+ logger.info("MU v2 experiment completed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ tokensIn,+ tokensOut,+ pages,+ });+ } catch (error) {+ const durationMs = Date.now() - startedAt;+ logger.warn("MU v2 experiment failed", { error, durationMs });+ }+ })();+ }+-+ const muV1StartedAt = Date.now();const podStart = await robustFetch({url:"https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",@@@ -276,6 -344,51 +341,53 @@@ export async function scrapePDF(meta: Mprocess.env.RUNPOD_MU_API_KEY &&process.env.RUNPOD_MU_POD_ID) {+ // Minimal experiment: fire MU v2 in background if enabled; do not await+ if (+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&+ process.env.PDF_MU_V2_BASE_URL+ ) {+ (async () => {+ const startedAt = Date.now();- const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });++ const logger = meta.logger.child({++ method: "scrapePDF/MUv2Experiment",++ });+ try {+ const resp = await robustFetch({+ url: process.env.PDF_MU_V2_BASE_URL ?? "",+ method: "POST",+ body: {+ input: {+ file_content: base64Content,+ filename: path.basename(tempFilePath) + ".pdf",+ timeout: meta.abort.scrapeTimeout(),+ created_at: Date.now(),+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ },+ logger,+ schema: z.any(),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ const body: any = resp as any;+ const tokensIn = body?.metadata?.["total-input-tokens"];+ const tokensOut = body?.metadata?.["total-output-tokens"];+ const pages = body?.metadata?.["pdf-total-pages"];+ const durationMs = Date.now() - startedAt;+ logger.info("MU v2 experiment completed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ tokensIn,+ tokensOut,+ pages,+ });+ } catch (error) {+ const durationMs = Date.now() - startedAt;+ logger.warn("MU v2 experiment failed", { error, durationMs });+ }+ })();+ }+ const muV1StartedAt = Date.now();try {result = await scrapePDFWithRunPodMU({diff --cc apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 3088a46c2,549a58221..73ea508d2--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@@ -129,8 -208,11 +208,17 @@@ const resolveRefs =const refPath = obj.$ref.split("/");if (refPath[0] === "#" && refPath[1] === "$defs") {const defName = refPath[refPath.length - 1];- return resolveRefs({ ...defs[defName] }, defs);+ if (defs[defName]) {- return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);++ return resolveRefs(++ { ...defs[defName] },++ defs,++ logger,++ visited,++ depth + 1,++ );+ }}+ return obj; // Return original if ref can't be resolved}// Handle arrays@@@ -193,11 -275,50 +281,58 @@@ export async function extractData(if (schema) {const defs = schema.$defs || {};- schema = resolveRefs(schema, defs);- delete schema.$defs;- logger.info("Resolved schema refs", {- schema,- });+ const schemaString = JSON.stringify(schema);+ const hasAnyRefs =+ schema.$defs ||+ schemaString.includes('"$ref"') ||+ schemaString.includes("#/$defs/");++ if (hasAnyRefs) {+ logger.info(+ "Detected schema with references, attempting to resolve refs",+ {+ hasDefsProperty: !!schema.$defs,+ hasRefInString: schemaString.includes('"$ref"'),+ hasRefPathInString: schemaString.includes("#/$defs/"),+ },+ );-+++ try {+ const resolvedSchema = resolveRefs(schema, defs, logger);++ const resolvedString = JSON.stringify(resolvedSchema);- const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");++ const hasRemainingRefs =++ resolvedString.includes('"$ref"') ||++ resolvedString.includes("#/$defs/");++ if (!hasRemainingRefs) {+ schema = resolvedSchema;- if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;++ if (schema && typeof schema === "object" && schema.$defs)++ delete schema.$defs;+ logger.info("Successfully resolved schema refs", {+ schema,+ });+ } else {- logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");++ logger.info(++ "Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema",++ );+ }+ } catch (error) {- logger.warn("Failed to resolve schema refs, preserving original schema", { error });++ logger.warn(++ "Failed to resolve schema refs, preserving original schema",++ { error },++ );+ }+ } else {+ logger.info("No recursive references detected, resolving refs", {+ schema,+ });+ schema = resolveRefs(schema, defs, logger);+ delete schema.$defs;+ logger.info("Resolved schema refs", {+ schema,+ });+ }}const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);