Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge remote-tracking branch 'origin/main' into devin/ENG-3639-1759244281

Ademílson Tonatoc57ad2d2d3ef
diff --cc apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
index b2f7c25ba,c60b12f4c..2a19c0368
--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
@@@ -1,189 -1,181 +1,184 @@@
+ import { describeIf, TEST_PRODUCTION } from "../lib";
import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";
- describe("V2 Scrape Screenshot Viewport", () => {
- if (!process.env.TEST_SUITE_SELF_HOSTED) {
- let identity: Identity;
-
- beforeAll(async () => {
- identity = await idmux({
- name: "v2-scrape-viewport",
- concurrency: 100,
- credits: 1000000,
- });
- }, 10000);
-
- test(
- "should take screenshot with custom viewport dimensions",
- async () => {
- const data = await scrape(
- {
- url: "https://example.com",
- formats: [
- "markdown",
- {
- type: "screenshot",
- viewport: {
- width: 1920,
- height: 1080,
- },
- },
- ],
- },
- identity,
- );
-
- expect(data).toBeDefined();
- expect(data.screenshot).toBeDefined();
- expect(data.markdown).toBeDefined();
- },
- scrapeTimeout,
- );
-
- test(
- "should reject fullPage=true with viewport",
- async () => {
- const response = await scrapeRaw(
- {
- url: "https://example.com",
- formats: [
- {
- type: "screenshot",
- fullPage: true,
- viewport: {
- width: 1440,
- height: 900,
- },
+ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "v2-scrape-viewport",
+ concurrency: 100,
+ credits: 1000000,
+ });
+ }, 10000);
+
+ test(
+ "should take screenshot with custom viewport dimensions",
+ async () => {
+ const data = await scrape(
+ {
+ url: "https://example.com",
+ formats: [
+ "markdown",
+ {
+ type: "screenshot",
+ viewport: {
+ width: 1920,
+ height: 1080,
},
- ],
- },
- identity,
- );
-
- expect(response.status).toBe(400);
- expect(response.body.success).toBe(false);
- expect(response.body.error).toContain(
- "Cannot specify viewport dimensions when fullPage is true",
- );
- },
- scrapeTimeout,
- );
-
- test(
- "should work with screenshot format without viewport (backwards compatibility)",
- async () => {
- const data = await scrape(
- {
- url: "https://example.com",
- formats: ["screenshot"],
- },
- identity,
- );
-
- expect(data).toBeDefined();
- expect(data.screenshot).toBeDefined();
- },
- scrapeTimeout,
- );
-
- test(
- "should work with object screenshot format without viewport",
- async () => {
- const data = await scrape(
- {
- url: "https://example.com",
- formats: [
- {
- type: "screenshot",
- fullPage: false,
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(data).toBeDefined();
+ expect(data.screenshot).toBeDefined();
+ expect(data.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ test(
- "should take full page screenshot with custom viewport width",
++ "should reject fullPage=true with viewport",
+ async () => {
- const data = await scrape(
++ const data = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "screenshot",
+ fullPage: true,
+ viewport: {
+ width: 1440,
+ height: 900,
},
- ],
- },
- identity,
- );
-
- expect(data).toBeDefined();
- expect(data.screenshot).toBeDefined();
- },
- scrapeTimeout,
- );
-
- test(
- "should reject invalid viewport dimensions",
- async () => {
- const response = await scrapeRaw(
- {
- url: "https://example.com",
- formats: [
- {
- type: "screenshot",
- viewport: {
- width: -100,
- height: 0,
- },
+ },
+ ],
+ },
+ identity,
+ );
+
- expect(data).toBeDefined();
- expect(data.screenshot).toBeDefined();
++ expect(response.status).toBe(400);
++ expect(response.body.success).toBe(false);
++ expect(response.body.error).toContain(
++ "Cannot specify viewport dimensions when fullPage is true",
++ );
+ },
+ scrapeTimeout,
+ );
+
+ test(
+ "should work with screenshot format without viewport (backwards compatibility)",
+ async () => {
+ const data = await scrape(
+ {
+ url: "https://example.com",
+ formats: ["screenshot"],
+ },
+ identity,
+ );
+
+ expect(data).toBeDefined();
+ expect(data.screenshot).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ test(
+ "should work with object screenshot format without viewport",
+ async () => {
+ const data = await scrape(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "screenshot",
+ fullPage: false,
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(data).toBeDefined();
+ expect(data.screenshot).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ test(
+ "should reject invalid viewport dimensions",
+ async () => {
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "screenshot",
+ viewport: {
+ width: -100,
+ height: 0,
},
- ],
- },
- identity,
- );
-
- expect(response.status).toBe(400);
- expect(response.body.success).toBe(false);
- },
- scrapeTimeout,
- );
-
- test(
- "should reject non-integer viewport dimensions",
- async () => {
- const response = await scrapeRaw(
- {
- url: "https://example.com",
- formats: [
- {
- type: "screenshot",
- viewport: {
- width: "1920" as any,
- height: 1080.5 as any,
- },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.status).toBe(400);
+ expect(response.body.success).toBe(false);
+ },
+ scrapeTimeout,
+ );
+
+ test(
+ "should reject non-integer viewport dimensions",
+ async () => {
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "screenshot",
+ viewport: {
+ width: "1920" as any,
+ height: 1080.5 as any,
},
- ],
- },
- identity,
- );
-
- expect(response.status).toBe(400);
- expect(response.body.success).toBe(false);
- },
- scrapeTimeout,
- );
-
- test(
- "should reject viewport dimensions exceeding maximum limits",
- async () => {
- const response = await scrapeRaw(
- {
- url: "https://example.com",
- formats: [
- {
- type: "screenshot",
- viewport: {
- width: 8000, // exceeds max of 7680
- height: 5000, // exceeds max of 4320
- },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.status).toBe(400);
+ expect(response.body.success).toBe(false);
+ },
+ scrapeTimeout,
+ );
+
+ test(
+ "should reject viewport dimensions exceeding maximum limits",
+ async () => {
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "screenshot",
+ viewport: {
+ width: 8000, // exceeds max of 7680
+ height: 5000, // exceeds max of 4320
},
- ],
- },
- identity,
- );
-
- expect(response.status).toBe(400);
- expect(response.body.success).toBe(false);
- },
- scrapeTimeout,
- );
- } else {
- it("mocked", () => {
- expect(true).toBe(true);
- });
- }
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.status).toBe(400);
+ expect(response.body.success).toBe(false);
+ },
+ scrapeTimeout,
+ );
});
diff --cc apps/api/src/controllers/v2/f-search.ts
index 000000000,67b56a2ee..80bc74b25
mode 000000,100644..100644
--- a/apps/api/src/controllers/v2/f-search.ts
+++ b/apps/api/src/controllers/v2/f-search.ts
@@@ -1,0 -1,230 +1,233 @@@
+ import { Request, Response } from "express";
+ import { z } from "zod";
+ import { logger as _logger } from "../../lib/logger";
+ import { CostTracking } from "../../lib/cost-tracking";
+ import {
+ getSearchIndexClient,
+ SearchIndexClient,
+ type SearchRequest,
+ } from "../../lib/search-index-client";
+
+ // Validation schemas
+ const searchRequestSchema = z.object({
+ query: z.string().min(1).max(500),
+ limit: z.number().int().min(1).max(100).optional().default(50),
+ offset: z.number().int().min(0).optional().default(0),
- mode: z.enum(["hybrid", "keyword", "semantic", "bm25"]).optional().default("hybrid"),
++ mode: z
++ .enum(["hybrid", "keyword", "semantic", "bm25"])
++ .optional()
++ .default("hybrid"),
+ filters: z
+ .object({
+ domain: z.string().optional(),
+ country: z.string().length(2).optional(),
+ isMobile: z.boolean().optional(),
+ minFreshness: z.number().min(0).max(1).optional(),
+ language: z.string().optional(),
+ })
+ .optional()
+ .default({}),
+ });
+
+ const searchChunksRequestSchema = z.object({
+ query: z.string().min(1).max(500),
+ limit: z.number().int().min(1).max(50).optional().default(20),
+ filters: z
+ .object({
+ domain: z.string().optional(),
+ })
+ .optional()
+ .default({}),
+ });
+
+ export async function realtimeSearchController(
+ req: Request,
+ res: Response,
+ ): Promise<void> {
+ const logger = _logger.child({
+ module: "realtime-search-controller",
+ method: "POST /admin/search",
+ teamId: (req as any).auth?.team_id,
+ });
+
+ try {
+ // Validate request
+ const validationResult = searchRequestSchema.safeParse(req.body);
+
+ if (!validationResult.success) {
+ res.status(400).json({
+ success: false,
+ error: "Invalid request parameters",
+ details: validationResult.error.errors,
+ });
+ return;
+ }
+
+ const { query, limit, offset, mode, filters } = validationResult.data;
+
+ logger.info("Search request", {
+ query,
+ mode,
+ limit,
+ filters,
+ });
+
+ // Get search index client
+ const client = getSearchIndexClient();
-
++
+ if (!client) {
+ res.status(503).json({
+ success: false,
+ error: "Search index service is not configured",
+ });
+ return;
+ }
+
+ // Perform search via HTTP client
+ const searchRequest: SearchRequest = {
+ query,
+ limit,
+ offset,
+ mode,
+ filters,
+ };
+
+ const result = await client.search(searchRequest, logger);
+
+ // Track cost (if applicable)
+ const costTracking = new CostTracking();
+ // Add search cost tracking here if needed
+
+ res.status(200).json({
+ success: true,
+ data: result,
+ costTracking: costTracking.toJSON(),
+ });
+ } catch (error) {
+ logger.error("Search request failed", {
+ error: (error as Error).message,
+ });
+
+ res.status(500).json({
+ success: false,
+ error: "Internal server error",
+ message: (error as Error).message,
+ });
+ }
+ }
+
+ // /**
+ // * POST /admin/search/chunks
+ // * Chunk-level search for precise snippets
+ // */
+ // export async function searchChunksController(
+ // req: Request,
+ // res: Response,
+ // ): Promise<void> {
+ // const logger = _logger.child({
+ // module: "search-chunks-controller",
+ // method: "POST /admin/search/chunks",
+ // teamId: (req as any).auth?.team_id,
+ // });
+
+ // try {
+ // // Validate request
+ // const validationResult = searchChunksRequestSchema.safeParse(req.body);
+
+ // if (!validationResult.success) {
+ // res.status(400).json({
+ // success: false,
+ // error: "Invalid request parameters",
+ // details: validationResult.error.errors,
+ // });
+ // return;
+ // }
+
+ // const { query, limit, filters } = validationResult.data;
+
+ // logger.info("Chunk search request", {
+ // query,
+ // limit,
+ // filters,
+ // });
+
+ // // Check if search index is enabled
+ // if (!isSearchIndexEnabled()) {
+ // res.status(503).json({
+ // success: false,
+ // error: "Search index is not configured",
+ // });
+ // return;
+ // }
+
+ // // Perform chunk search
+ // const results = await searchChunks(
+ // search_index_supabase_service,
+ // query,
+ // limit,
+ // filters as SearchFilters,
+ // logger,
+ // );
+
+ // res.status(200).json({
+ // success: true,
+ // data: {
+ // chunks: results,
+ // total: results.length,
+ // query,
+ // },
+ // });
+ // } catch (error) {
+ // logger.error("Chunk search request failed", {
+ // error: (error as Error).message,
+ // });
+
+ // res.status(500).json({
+ // success: false,
+ // error: "Internal server error",
+ // message: (error as Error).message,
+ // });
+ // }
+ // }
+
+ // /**
+ // * GET /admin/search/stats
+ // * Get search index statistics
+ // */
+ // export async function searchStatsController(
+ // req: Request,
+ // res: Response,
+ // ): Promise<void> {
+ // const logger = _logger.child({
+ // module: "search-stats-controller",
+ // method: "GET /admin/search/stats",
+ // });
+
+ // try {
+ // // Check if search index is enabled
+ // if (!isSearchIndexEnabled()) {
+ // res.status(503).json({
+ // success: false,
+ // error: "Search index is not configured",
+ // });
+ // return;
+ // }
+
+ // const stats = await getSearchStats(search_index_supabase_service);
+
+ // res.status(200).json({
+ // success: true,
+ // data: stats,
+ // });
+ // } catch (error) {
+ // logger.error("Failed to get search stats", {
+ // error: (error as Error).message,
+ // });
+
+ // res.status(500).json({
+ // success: false,
+ // error: "Internal server error",
+ // message: (error as Error).message,
+ // });
+ // }
+ // }
diff --cc apps/api/src/lib/crawl-redis.ts
index e0b777d8a,ff4ee837f..1b256ea67
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@@ -49,6 -49,24 +49,18 @@@ export async function saveCrawl(id: str
});
}
-export async function recordRobotsBlocked(
- crawlId: string,
- url: string,
-) {
- await redisEvictConnection.sadd(
- "crawl:" + crawlId + ":robots_blocked",
- url,
- );
++export async function recordRobotsBlocked(crawlId: string, url: string) {
++ await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);
+ await redisEvictConnection.expire(
+ "crawl:" + crawlId + ":robots_blocked",
+ 24 * 60 * 60,
+ );
+ }
+
+ export async function markCrawlActive(id: string) {
+ await redisEvictConnection.sadd("active_crawls", id);
+ }
+
export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {
return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);
}
diff --cc apps/api/src/lib/search-query-builder.ts
index 613d0fd86,9619d1ec8..ecfc43cf4
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@@ -116,6 -134,8 +134,8 @@@ export function getCategoryFromUrl
// Check against category map for other sites
for (const [site, category] of categoryMap.entries()) {
+ if (site === "__pdf__") continue; // Skip the special PDF marker
-
++
if (
hostname === site.toLowerCase() ||
hostname.endsWith("." + site.toLowerCase())
diff --cc apps/api/src/scraper/scrapeURL/engines/document/index.ts
index 649b520c0,8a8cfc8f4..f56d23fb8
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@@ -41,30 -65,94 +65,98 @@@ function getDocumentTypeFromContentType
return null;
}
+ function isValidDocumentContentType(contentType: string | null): boolean {
+ if (!contentType) return false;
+
+ const ct = contentType.toLowerCase();
+ const validTypes = [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel",
+ "application/msword",
+ "application/rtf",
+ "text/rtf",
+ "application/vnd.oasis.opendocument.text",
+ ];
+
+ return validTypes.some(type => ct.includes(type));
+ }
+
export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
- const { response, buffer } = await fetchFileToBuffer(
- meta.rewrittenUrl ?? meta.url,
- {
+ let response: Response;
+ let buffer: Buffer;
+ let proxyUsed: "basic" | "stealth" = "basic";
+ let tempFilePath: string | null = null;
+
+ if (meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
+ // Use prefetched document
+ tempFilePath = meta.documentPrefetch.filePath;
+ buffer = await readFile(tempFilePath);
+
+ // Create a mock response object with content-type from prefetch
+ const headers = new Headers();
+ if (meta.documentPrefetch.contentType) {
+ headers.set("Content-Type", meta.documentPrefetch.contentType);
+ }
+
+ response = {
+ url: meta.documentPrefetch.url ?? meta.rewrittenUrl ?? meta.url,
+ status: meta.documentPrefetch.status,
+ headers,
+ } as Response;
+
+ proxyUsed = meta.documentPrefetch.proxyUsed;
+ } else {
+ // Fetch the document normally
+ const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
headers: meta.options.headers,
signal: meta.abort.asSignal(),
- },
- );
-
- const documentType =
- getDocumentTypeFromContentType(response.headers.get("content-type")) ??
- getDocumentTypeFromUrl(response.url);
-
- const html = await converter.convertBufferToHtml(
- new Uint8Array(buffer),
- documentType,
- );
-
- return {
- url: response.url,
- statusCode: response.status,
- html,
- proxyUsed: "basic",
- };
+ });
+ response = result.response;
+ buffer = result.buffer;
+
+ // Validate content type only when fetching directly (not using prefetch)
+ const ct = response.headers.get("Content-Type");
+ if (ct && !isValidDocumentContentType(ct)) {
+ // if downloaded file wasn't a valid document, throw antibot error
+ throw new DocumentAntibotError();
+ }
+ }
+
+ try {
+ const documentType =
+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??
+ getDocumentTypeFromUrl(response.url);
+
+ const html = await converter.convertBufferToHtml(
+ new Uint8Array(buffer),
+ documentType,
+ );
+
+ return {
+ url: response.url,
+ statusCode: response.status,
+ html,
+ proxyUsed,
+ };
+ } finally {
+ // Clean up temporary file if it was created by prefetch
- if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
++ if (
++ tempFilePath &&
++ meta.documentPrefetch !== undefined &&
++ meta.documentPrefetch !== null
++ ) {
+ try {
+ await unlink(tempFilePath);
+ } catch (error) {
+ // Ignore errors when cleaning up temp files
- meta.logger?.warn("Failed to clean up temporary document file", {
- error,
- tempFilePath
++ meta.logger?.warn("Failed to clean up temporary document file", {
++ error,
++ tempFilePath,
+ });
+ }
+ }
+ }
}
export function documentMaxReasonableTime(meta: Meta): number {
diff --cc apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 0f6442e53,7b300fac4..00e7c713e
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@@ -67,6 -67,56 +67,53 @@@ async function scrapePDFWithRunPodMU
maxPages,
});
-
-
+ if (
+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&
+ process.env.PDF_MU_V2_BASE_URL &&
+ Math.random() * 100 <
+ Number(process.env.PDF_MU_V2_EXPERIMENT_PERCENT ?? "100")
+ ) {
+ (async () => {
+ const startedAt = Date.now();
+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
+ try {
+ const resp = await robustFetch({
+ url: process.env.PDF_MU_V2_BASE_URL ?? "",
+ method: "POST",
+ body: {
+ input: {
+ file_content: base64Content,
+ filename: path.basename(tempFilePath) + ".pdf",
+ timeout: meta.abort.scrapeTimeout(),
+ created_at: Date.now(),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ },
+ logger,
+ schema: z.any(),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ const body: any = resp as any;
+ const tokensIn = body?.metadata?.["total-input-tokens"];
+ const tokensOut = body?.metadata?.["total-output-tokens"];
+ const pages = body?.metadata?.["pdf-total-pages"];
+ const durationMs = Date.now() - startedAt;
+ logger.info("MU v2 experiment completed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ tokensIn,
+ tokensOut,
+ pages,
+ });
+ } catch (error) {
+ const durationMs = Date.now() - startedAt;
+ logger.warn("MU v2 experiment failed", { error, durationMs });
+ }
+ })();
+ }
+
-
+ const muV1StartedAt = Date.now();
const podStart = await robustFetch({
url:
"https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",
@@@ -276,6 -344,51 +341,53 @@@ export async function scrapePDF(meta: M
process.env.RUNPOD_MU_API_KEY &&
process.env.RUNPOD_MU_POD_ID
) {
+ // Minimal experiment: fire MU v2 in background if enabled; do not await
+ if (
+ process.env.PDF_MU_V2_EXPERIMENT === "true" &&
+ process.env.PDF_MU_V2_BASE_URL
+ ) {
+ (async () => {
+ const startedAt = Date.now();
- const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
++ const logger = meta.logger.child({
++ method: "scrapePDF/MUv2Experiment",
++ });
+ try {
+ const resp = await robustFetch({
+ url: process.env.PDF_MU_V2_BASE_URL ?? "",
+ method: "POST",
+ body: {
+ input: {
+ file_content: base64Content,
+ filename: path.basename(tempFilePath) + ".pdf",
+ timeout: meta.abort.scrapeTimeout(),
+ created_at: Date.now(),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ },
+ logger,
+ schema: z.any(),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ const body: any = resp as any;
+ const tokensIn = body?.metadata?.["total-input-tokens"];
+ const tokensOut = body?.metadata?.["total-output-tokens"];
+ const pages = body?.metadata?.["pdf-total-pages"];
+ const durationMs = Date.now() - startedAt;
+ logger.info("MU v2 experiment completed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ tokensIn,
+ tokensOut,
+ pages,
+ });
+ } catch (error) {
+ const durationMs = Date.now() - startedAt;
+ logger.warn("MU v2 experiment failed", { error, durationMs });
+ }
+ })();
+ }
+ const muV1StartedAt = Date.now();
try {
result = await scrapePDFWithRunPodMU(
{
diff --cc apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 3088a46c2,549a58221..73ea508d2
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@@ -129,8 -208,11 +208,17 @@@ const resolveRefs =
const refPath = obj.$ref.split("/");
if (refPath[0] === "#" && refPath[1] === "$defs") {
const defName = refPath[refPath.length - 1];
- return resolveRefs({ ...defs[defName] }, defs);
+ if (defs[defName]) {
- return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);
++ return resolveRefs(
++ { ...defs[defName] },
++ defs,
++ logger,
++ visited,
++ depth + 1,
++ );
+ }
}
+ return obj; // Return original if ref can't be resolved
}
// Handle arrays
@@@ -193,11 -275,50 +281,58 @@@ export async function extractData(
if (schema) {
const defs = schema.$defs || {};
- schema = resolveRefs(schema, defs);
- delete schema.$defs;
- logger.info("Resolved schema refs", {
- schema,
- });
+ const schemaString = JSON.stringify(schema);
+ const hasAnyRefs =
+ schema.$defs ||
+ schemaString.includes('"$ref"') ||
+ schemaString.includes("#/$defs/");
+
+ if (hasAnyRefs) {
+ logger.info(
+ "Detected schema with references, attempting to resolve refs",
+ {
+ hasDefsProperty: !!schema.$defs,
+ hasRefInString: schemaString.includes('"$ref"'),
+ hasRefPathInString: schemaString.includes("#/$defs/"),
+ },
+ );
-
++
+ try {
+ const resolvedSchema = resolveRefs(schema, defs, logger);
+
+ const resolvedString = JSON.stringify(resolvedSchema);
- const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");
++ const hasRemainingRefs =
++ resolvedString.includes('"$ref"') ||
++ resolvedString.includes("#/$defs/");
+
+ if (!hasRemainingRefs) {
+ schema = resolvedSchema;
- if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;
++ if (schema && typeof schema === "object" && schema.$defs)
++ delete schema.$defs;
+ logger.info("Successfully resolved schema refs", {
+ schema,
+ });
+ } else {
- logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");
++ logger.info(
++ "Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema",
++ );
+ }
+ } catch (error) {
- logger.warn("Failed to resolve schema refs, preserving original schema", { error });
++ logger.warn(
++ "Failed to resolve schema refs, preserving original schema",
++ { error },
++ );
+ }
+ } else {
+ logger.info("No recursive references detected, resolving refs", {
+ schema,
+ });
+ schema = resolveRefs(schema, defs, logger);
+ delete schema.$defs;
+ logger.info("Resolved schema refs", {
+ schema,
+ });
+ }
}
const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);