Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
more test fixes
Gergő Móricza0a22cb459a8
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex a200f2e77..1b43809ae 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -187,7 +187,7 @@ export async function batchScrapeController(crawlId: id,data: null,webhook: req.body.webhook,- v1: false,+ v1: true,eventType: "batch_scrape.started",});}diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.tsnew file mode 100644index 000000000..42cfe07d6--- /dev/null+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts@@ -0,0 +1,35 @@+import { Response } from "express";+import {+ OngoingCrawlsResponse,+ RequestWithAuth,+ toV2CrawlerOptions,+} from "./types";+import {+ getCrawl,+ getCrawlsByTeamId,+} from "../../lib/crawl-redis";+import { configDotenv } from "dotenv";+configDotenv();++export async function ongoingCrawlsController(+ req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,+ res: Response<OngoingCrawlsResponse>,+) {+ const ids = await getCrawlsByTeamId(req.auth.team_id);++ const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))).filter((crawl) => crawl !== null && !crawl.cancelled && crawl.crawlerOptions);++ res.status(200).json({+ success: true,+ crawls: crawls.map(x => ({+ id: x.id,+ teamId: x.team_id!,+ url: x.originUrl!,+ created_at: new Date(x.createdAt || Date.now()).toISOString(),+ options: {+ ...toV2CrawlerOptions(x.crawlerOptions),+ scrapeOptions: x.scrapeOptions,+ },+ })),+ });+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 99d74b68c..deea491c9 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -9,6 +9,7 @@ import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preimport { crawlStatusController } from "../controllers/v2/crawl-status";import { mapV2Controller } from "../controllers/v2/map";import { crawlErrorsController } from "../controllers/v2/crawl-errors";+import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";import {authMiddleware,checkCreditsMiddleware,@@ -82,3 +83,9 @@ v2Router.get(authMiddleware(RateLimiterMode.CrawlStatus),wrap(crawlErrorsController),);++v2Router.get(+ "/crawl/ongoing",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(ongoingCrawlsController),+);\ No newline at end of filediff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex ae0071070..134201c85 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -283,7 +283,7 @@ export const callWebhook = async ({});}} catch (error) {- logger.debug(+ logger.warn(`Error sending webhook`,{error,