Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix types
Gergő Móriczf5abba801497
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 31d58eff6..dec7485e6 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -22,6 +22,7 @@ import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";import { supabase_service } from "../../services/supabase";+import { fromV1ScrapeOptions } from "../v2/types";interface DocumentWithCostTracking {document: Document;@@ -101,16 +102,17 @@ async function scrapeX402SearchResult(origin: options.origin,zeroDataRetention,});+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(options.scrapeOptions, options.timeout, options.teamId);await addScrapeJob({url: searchResult.url,mode: "single_urls" as Mode,team_id: options.teamId,scrapeOptions: {- ...options.scrapeOptions,- maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days+ ...scrapeOptions,+ maxAge: scrapeOptions.maxAge === 0 ? 3 * 24 * 60 * 60 * 1000 : scrapeOptions.maxAge,},- internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },+ internalOptions: { ...internalOptions, teamId: options.teamId, bypassBilling: true, zeroDataRetention },origin: options.origin,is_scrape: true,startTime: Date.now(),diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex b951784e9..4199f910e 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -61,7 +61,8 @@ export async function crawlParamsPreviewController(const { extract } = await generateCrawlerOptionsFromPrompt(parsedBody.prompt,logger,- costTracking+ costTracking,+ { teamId: req.auth.team_id });const generatedOptions = extract || {};diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 04c4a798d..10b8922dc 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -68,7 +68,8 @@ export async function crawlController(const { extract } = await generateCrawlerOptionsFromPrompt(req.body.prompt,logger,- costTracking+ costTracking,+ { teamId: req.auth.team_id, crawlId: id });promptGeneratedOptions = extract || {};logger.debug("Generated crawler options from prompt", {diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 3fe3ffc8d..b20250320 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -6,13 +6,11 @@ import { EngineError, IndexMissError, TimeoutError } from "../../error";import crypto from "crypto";export async function sendDocumentToIndex(meta: Meta, document: Document) {--const shouldCache = meta.options.storeInCache&& !meta.internalOptions.zeroDataRetention&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"- && !(meta.winnerEngine === "pdf" && meta.options.parsePDF === false)+ && !(meta.winnerEngine === "pdf" && meta.options.parsers?.includes("pdf") === false)&& (meta.internalOptions.teamId === "sitemap"|| (@@ -212,13 +210,13 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefinconst isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)- if (isCachedPdfBase64 && meta.options.parsePDF !== false) {+ if (isCachedPdfBase64 && meta.options.parsers?.includes("pdf") !== false) {// Cached content is unparsed PDF, but we want parsed - report cache missthrow new IndexMissError();}// If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)- if (!isCachedPdfBase64 && meta.options.parsePDF === false) {+ if (!isCachedPdfBase64 && meta.options.parsers?.includes("pdf") === false) {// Check if URL looks like a PDFconst isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");if (isPdfUrl) {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 34b1252f4..25088db8c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -612,8 +612,8 @@ export async function scrapeURL(&& abRate > 0&& Math.random() <= abRate&& abHostEnv- && meta.options.agent === undefined- && (meta.options.extract || meta.options.jsonOptions)?.agent === undefined;+ && meta.internalOptions.v1Agent === undefined+ && meta.internalOptions.v1JSONAgent === undefined;if (shouldABTest) {(async () => {try {@@ -621,7 +621,7 @@ export async function scrapeURL(abLogger.info("A/B-testing scrapeURL to staging");const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));await robustFetch({- url: `http://${abHostEnv}/v1/scrape`,+ url: `http://${abHostEnv}/v2/scrape`,method: "POST",body: {url: meta.url,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 71a1215a3..98ee0ffde 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -957,6 +957,11 @@ export async function performSummary(method: "performSummary",},},+ metadata: {+ teamId: meta.internalOptions.teamId,+ functionId: "performSummary",+ scrapeId: meta.id,+ }};const {@@ -1109,6 +1114,7 @@ export async function generateCrawlerOptionsFromPrompt(prompt: string,logger: Logger,costTracking: CostTracking,+ metadata: { teamId: string, crawlId?: string },): Promise<{ extract: any }> {const model = getModel("gpt-4o", "openai");const retryModel = getModel("gpt-4o-mini", "openai");@@ -1151,6 +1157,10 @@ Return a JSON object with only the relevant options for the user's request. Don'method: "generateCrawlerOptionsFromPrompt",},},+ metadata: {+ ...metadata,+ functionId: "generateCrawlerOptionsFromPrompt",+ }});return { extract };