Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq): further indexes
Gergő Móriczc2ddb24a0263
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 61af944bd..76fd929bb 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -54,6 +54,9 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_mode_status_idx ON nuq.queue_s-- For getCrawlJobsForListing: query by group_id, status='completed', data->>'mode', ordered by finished_at, created_atCREATE INDEX IF NOT EXISTS nuq_queue_scrape_group_completed_listing_idx ON nuq.queue_scrape (group_id, finished_at ASC, created_at ASC) WHERE (status = 'completed'::nuq.job_status AND (data->>'mode') = 'single_urls');+-- For group finish cron+CREATE INDEX IF NOT EXISTS idx_queue_scrape_group_status ON nuq.queue_scrape (group_id, status) WHERE status IN ('active', 'queued');+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog (id uuid NOT NULL DEFAULT gen_random_uuid(),data jsonb,@@ -174,8 +177,8 @@ SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$WITH cleaned_groups AS (DELETE FROM nuq.group_crawlWHERE nuq.group_crawl.status = 'completed'::nuq.group_status- AND nuq.group_crawl.expires_at < now();- RETURNING *;+ AND nuq.group_crawl.expires_at < now()+ RETURNING *), cleaned_jobs_queue_scrape AS (DELETE FROM nuq.queue_scrapeWHERE nuq.queue_scrape.group_id IN (SELECT id FROM cleaned_groups)@@ -185,5 +188,6 @@ SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$), cleaned_jobs_crawl_finished AS (DELETE FROM nuq.queue_crawl_finishedWHERE nuq.queue_crawl_finished.group_id IN (SELECT id FROM cleaned_groups)- );+ )+ SELECT 1;$$);