Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

more test fixes

Gergő Móricza0a22cb459a8
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index a200f2e77..1b43809ae 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -187,7 +187,7 @@ export async function batchScrapeController(
crawlId: id,
data: null,
webhook: req.body.webhook,
- v1: false,
+ v1: true,
eventType: "batch_scrape.started",
});
}
diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.ts
new file mode 100644
index 000000000..42cfe07d6
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts
@@ -0,0 +1,35 @@
+import { Response } from "express";
+import {
+ OngoingCrawlsResponse,
+ RequestWithAuth,
+ toV2CrawlerOptions,
+} from "./types";
+import {
+ getCrawl,
+ getCrawlsByTeamId,
+} from "../../lib/crawl-redis";
+import { configDotenv } from "dotenv";
+configDotenv();
+
+export async function ongoingCrawlsController(
+ req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,
+ res: Response<OngoingCrawlsResponse>,
+) {
+ const ids = await getCrawlsByTeamId(req.auth.team_id);
+
+ const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))).filter((crawl) => crawl !== null && !crawl.cancelled && crawl.crawlerOptions);
+
+ res.status(200).json({
+ success: true,
+ crawls: crawls.map(x => ({
+ id: x.id,
+ teamId: x.team_id!,
+ url: x.originUrl!,
+ created_at: new Date(x.createdAt || Date.now()).toISOString(),
+ options: {
+ ...toV2CrawlerOptions(x.crawlerOptions),
+ scrapeOptions: x.scrapeOptions,
+ },
+ })),
+ });
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 99d74b68c..deea491c9 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -9,6 +9,7 @@ import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-pre
import { crawlStatusController } from "../controllers/v2/crawl-status";
import { mapV2Controller } from "../controllers/v2/map";
import { crawlErrorsController } from "../controllers/v2/crawl-errors";
+import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -82,3 +83,9 @@ v2Router.get(
authMiddleware(RateLimiterMode.CrawlStatus),
wrap(crawlErrorsController),
);
+
+v2Router.get(
+ "/crawl/ongoing",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(ongoingCrawlsController),
+);
\ No newline at end of file
diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts
index ae0071070..134201c85 100644
--- a/apps/api/src/services/webhook.ts
+++ b/apps/api/src/services/webhook.ts
@@ -283,7 +283,7 @@ export const callWebhook = async ({
});
}
} catch (error) {
- logger.debug(
+ logger.warn(
`Error sending webhook`,
{
error,