Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: various crawl things
Gergő Móricz76c7be67e808
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 15ed8ac7c..5a80cc665 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -268,7 +268,8 @@ export async function crawlStatusController(outputBulkB = {data: scrapes,next:- (outputBulkA.total ?? 0) > start + iteratedOver+ (outputBulkA.total ?? 0) > start + iteratedOver ||+ outputBulkA.status !== "completed"? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 551cf9095..8f4688993 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -280,7 +280,8 @@ export async function crawlStatusController(outputBulkB = {data: scrapes,next:- (outputBulkA.total ?? 0) > start + iteratedOver+ (outputBulkA.total ?? 0) > start + iteratedOver ||+ outputBulkA.status !== "completed"? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex 286265a6b..8137162cf 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -1,35 +1,53 @@import "dotenv/config";-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck, nuqShutdown } from "./nuq";+import {+ scrapeQueue,+ nuqGetLocalMetrics,+ nuqHealthCheck,+ nuqShutdown,+ crawlFinishedQueue,+} from "./nuq";import Express from "express";import { logger } from "../../lib/logger";(async () => {- const app = Express();+ const app = Express();- app.get("/metrics", (_, res) => res.contentType("text/plain").send(nuqGetLocalMetrics()));- app.get("/health", async (_, res) => {- if (await nuqHealthCheck()) {- res.status(200).send("OK");- } else {- res.status(500).send("Not OK");- }- });+ app.get("/metrics", (_, res) =>+ res.contentType("text/plain").send(nuqGetLocalMetrics()),+ );+ app.get("/health", async (_, res) => {+ if (await nuqHealthCheck()) {+ res.status(200).send("OK");+ } else {+ res.status(500).send("Not OK");+ }+ });- const server = app.listen(Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011), () => {- logger.info("NuQ prefetch worker metrics server started");- });+ const server = app.listen(+ Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),+ () => {+ logger.info("NuQ prefetch worker metrics server started");+ },+ );- async function shutdown() {- server.close();- await nuqShutdown();- process.exit(0);- }+ async function shutdown() {+ server.close();+ await nuqShutdown();+ process.exit(0);+ }- process.on("SIGINT", shutdown);- process.on("SIGTERM", shutdown);+ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);+ (async () => {while (true) {- await scrapeQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 250));+ await crawlFinishedQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 250));}+ })();++ while (true) {+ await scrapeQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 250));+ }})();diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 119ec8d27..0de84d644 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -44,6 +44,19 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scCREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);+-- Indexes for crawl-status.ts queries+-- For getGroupAnyJob: query by group_id, owner_id, and data->>'mode' = 'single_urls'+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_owner_mode_idx ON nuq.queue_scrape (group_id, owner_id) WHERE ((data->>'mode') = 'single_urls');++-- For getGroupNumericStats: query by group_id and data->>'mode', grouped by status+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_mode_status_idx ON nuq.queue_scrape (group_id, status) WHERE ((data->>'mode') = 'single_urls');++-- For getCrawlJobsForListing: query by group_id, status='completed', data->>'mode', ordered by finished_at, created_at+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_completed_listing_idx ON nuq.queue_scrape (group_id, finished_at ASC, created_at ASC) WHERE (status = 'completed'::nuq.job_status AND (data->>'mode') = 'single_urls');++-- For getGroupNumericStats backlog query: query by group_id and data->>'mode' on backlog table+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_group_mode_idx ON nuq.queue_scrape_backlog (group_id) WHERE ((data->>'mode') = 'single_urls');+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog (id uuid NOT NULL DEFAULT gen_random_uuid(),data jsonb,