Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: various crawl things

Gergő Móricz76c7be67e808
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 15ed8ac7c..5a80cc665 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -268,7 +268,8 @@ export async function crawlStatusController(
outputBulkB = {
data: scrapes,
next:
- (outputBulkA.total ?? 0) > start + iteratedOver
+ (outputBulkA.total ?? 0) > start + iteratedOver ||
+ outputBulkA.status !== "completed"
? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 551cf9095..8f4688993 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -280,7 +280,8 @@ export async function crawlStatusController(
outputBulkB = {
data: scrapes,
next:
- (outputBulkA.total ?? 0) > start + iteratedOver
+ (outputBulkA.total ?? 0) > start + iteratedOver ||
+ outputBulkA.status !== "completed"
? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
: undefined,
};
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.ts
index 286265a6b..8137162cf 100644
--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts
+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts
@@ -1,35 +1,53 @@
import "dotenv/config";
-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck, nuqShutdown } from "./nuq";
+import {
+ scrapeQueue,
+ nuqGetLocalMetrics,
+ nuqHealthCheck,
+ nuqShutdown,
+ crawlFinishedQueue,
+} from "./nuq";
import Express from "express";
import { logger } from "../../lib/logger";
(async () => {
- const app = Express();
+ const app = Express();
- app.get("/metrics", (_, res) => res.contentType("text/plain").send(nuqGetLocalMetrics()));
- app.get("/health", async (_, res) => {
- if (await nuqHealthCheck()) {
- res.status(200).send("OK");
- } else {
- res.status(500).send("Not OK");
- }
- });
+ app.get("/metrics", (_, res) =>
+ res.contentType("text/plain").send(nuqGetLocalMetrics()),
+ );
+ app.get("/health", async (_, res) => {
+ if (await nuqHealthCheck()) {
+ res.status(200).send("OK");
+ } else {
+ res.status(500).send("Not OK");
+ }
+ });
- const server = app.listen(Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011), () => {
- logger.info("NuQ prefetch worker metrics server started");
- });
+ const server = app.listen(
+ Number(process.env.NUQ_PREFETCH_WORKER_PORT ?? process.env.PORT ?? 3011),
+ () => {
+ logger.info("NuQ prefetch worker metrics server started");
+ },
+ );
- async function shutdown() {
- server.close();
- await nuqShutdown();
- process.exit(0);
- }
+ async function shutdown() {
+ server.close();
+ await nuqShutdown();
+ process.exit(0);
+ }
- process.on("SIGINT", shutdown);
- process.on("SIGTERM", shutdown);
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
+ (async () => {
while (true) {
- await scrapeQueue.prefetchJobs();
- await new Promise(resolve => setTimeout(resolve, 250));
+ await crawlFinishedQueue.prefetchJobs();
+ await new Promise(resolve => setTimeout(resolve, 250));
}
+ })();
+
+ while (true) {
+ await scrapeQueue.prefetchJobs();
+ await new Promise(resolve => setTimeout(resolve, 250));
+ }
})();
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 119ec8d27..0de84d644 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -44,6 +44,19 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_sc
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);
+-- Indexes for crawl-status.ts queries
+-- For getGroupAnyJob: query by group_id, owner_id, and data->>'mode' = 'single_urls'
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_owner_mode_idx ON nuq.queue_scrape (group_id, owner_id) WHERE ((data->>'mode') = 'single_urls');
+
+-- For getGroupNumericStats: query by group_id and data->>'mode', grouped by status
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_mode_status_idx ON nuq.queue_scrape (group_id, status) WHERE ((data->>'mode') = 'single_urls');
+
+-- For getCrawlJobsForListing: query by group_id, status='completed', data->>'mode', ordered by finished_at, created_at
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_completed_listing_idx ON nuq.queue_scrape (group_id, finished_at ASC, created_at ASC) WHERE (status = 'completed'::nuq.job_status AND (data->>'mode') = 'single_urls');
+
+-- For getGroupNumericStats backlog query: query by group_id and data->>'mode' on backlog table
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_group_mode_idx ON nuq.queue_scrape_backlog (group_id) WHERE ((data->>'mode') = 'single_urls');
+
CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog (
id uuid NOT NULL DEFAULT gen_random_uuid(),
data jsonb,