Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into nsc/v2
Gergő Móricz17008acee3fe
diff --cc apps/api/src/controllers/v0/crawl.tsindex 6b0eb9796,65e5c4ed1..cf0421330--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@@ -160,9 -158,7 +159,7 @@@ export async function crawlController(r// }// }- await logCrawl(id, team_id);-- const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(+ const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(pageOptions,undefined,undefined,diff --cc apps/api/src/controllers/v1/batch-scrape.tsindex 2f8a3e5ed,57f6c4499..fef7033be--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@@ -97,12 -95,6 +96,8 @@@ export async function batchScrapeControaccount: req.account,});- if (!req.body.appendToId) {- await logCrawl(id, req.auth.team_id);- }-+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);+const sc: StoredCrawl = req.body.appendToId? ((await getCrawl(req.body.appendToId)) as StoredCrawl): {diff --cc apps/api/src/controllers/v1/crawl.tsindex c7a87c373,ab5d2decd..62df33703--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@@ -8,10 -8,8 +8,9 @@@ importtoLegacyCrawlerOptions,} from "./types";import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";- import { logCrawl } from "../../services/logging/crawl_log";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";+import { fromV1ScrapeOptions } from "../v2/types";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,diff --cc apps/api/src/controllers/v1/types.tsindex 05c01156d,8571612ee..5e61566ce--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@@ -793,8 -794,8 +793,9 @@@ export const crawlRequestSchema = crawlexport type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;+// Note: Map types have been transitioned to v2/types.ts while maintaining backwards compatibilityexport const mapRequestSchema = crawlerOptions+ .omit({ ignoreQueryParameters: true }).extend({url,origin: z.string().optional().default("api"),diff --cc apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex b86d1cb2e,420e632b2..fcd1ce0af--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@@ -49,9 -52,16 +49,10 @@@ async function scrapePDFWithRunPodMU});}- const timeout = timeToRun- ? timeToRun - (Date.now() - preCacheCheckStartTime)- : undefined;- if (timeout && timeout < 0) {- throw new TimeoutError("MU PDF parser already timed out before call");- }+ meta.abort.throwIfAborted();- const abort = timeout ? AbortSignal.timeout(timeout) : undefined;+const podStart = await robustFetch({url:"https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",