Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge branch 'main' into nsc/v2

Gergő Móricz17008acee3fe
diff --cc apps/api/src/controllers/v0/crawl.ts
index 6b0eb9796,65e5c4ed1..cf0421330
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@@ -160,9 -158,7 +159,7 @@@ export async function crawlController(r
// }
// }
- await logCrawl(id, team_id);
-
- const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
+ const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(
pageOptions,
undefined,
undefined,
diff --cc apps/api/src/controllers/v1/batch-scrape.ts
index 2f8a3e5ed,57f6c4499..fef7033be
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@@ -97,12 -95,6 +96,8 @@@ export async function batchScrapeContro
account: req.account,
});
- if (!req.body.appendToId) {
- await logCrawl(id, req.auth.team_id);
- }
-
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);
+
const sc: StoredCrawl = req.body.appendToId
? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
: {
diff --cc apps/api/src/controllers/v1/crawl.ts
index c7a87c373,ab5d2decd..62df33703
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@@ -8,10 -8,8 +8,9 @@@ import
toLegacyCrawlerOptions,
} from "./types";
import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
- import { logCrawl } from "../../services/logging/crawl_log";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
+import { fromV1ScrapeOptions } from "../v2/types";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
diff --cc apps/api/src/controllers/v1/types.ts
index 05c01156d,8571612ee..5e61566ce
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@@ -793,8 -794,8 +793,9 @@@ export const crawlRequestSchema = crawl
export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
+// Note: Map types have been transitioned to v2/types.ts while maintaining backwards compatibility
export const mapRequestSchema = crawlerOptions
+ .omit({ ignoreQueryParameters: true })
.extend({
url,
origin: z.string().optional().default("api"),
diff --cc apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index b86d1cb2e,420e632b2..fcd1ce0af
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@@ -49,9 -52,16 +49,10 @@@ async function scrapePDFWithRunPodMU
});
}
- const timeout = timeToRun
- ? timeToRun - (Date.now() - preCacheCheckStartTime)
- : undefined;
- if (timeout && timeout < 0) {
- throw new TimeoutError("MU PDF parser already timed out before call");
- }
+ meta.abort.throwIfAborted();
- const abort = timeout ? AbortSignal.timeout(timeout) : undefined;
+
const podStart = await robustFetch({
url:
"https://api.runpod.ai/v2/" + process.env.RUNPOD_MU_POD_ID + "/runsync",