Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq-postgres): spread reindexes, bound maintenance with statement_timeout (#3467)
firecrawl-spring[bot]635a836c1040
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 1e8fb60e9..742a336f2 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -105,6 +105,9 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_owner_id_idx ON nuq.queue_sc-- For getGroupNumericStats backlog query: query by group_id and data->>'mode' on backlog tableCREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_group_mode_idx ON nuq.queue_scrape_backlog (group_id) WHERE ((data->>'mode') = 'single_urls');+-- For nuq_queue_scrape_backlog_reaper: avoid seq scan over the entire backlog every minute+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_times_out_at_idx ON nuq.queue_scrape_backlog (times_out_at);+SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour' AND group_id IS NULL;$$);@@ -120,12 +123,47 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$$$);SELECT cron.schedule('nuq_queue_scrape_backlog_reaper', '* * * * *', $$+ SET statement_timeout = '50s';DELETE FROM nuq.queue_scrape_backlogWHERE nuq.queue_scrape_backlog.times_out_at < now();$$);-SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$- REINDEX TABLE CONCURRENTLY nuq.queue_scrape;+-- Per-index REINDEX CONCURRENTLY, spread across the global low-traffic window+-- (02:00-06:20 UTC). REINDEX INDEX takes a ShareUpdateExclusiveLock only on+-- the target index, not the whole table.+--+-- Note: REINDEX CONCURRENTLY cannot run inside a transaction block, and pg_cron+-- wraps multi-statement commands in an implicit transaction. So each cron body+-- must be a single REINDEX statement -- statement_timeout cannot be set+-- inline. Stuck reindexes are caught by nuq_maintenance_watchdog below.+SELECT cron.schedule('nuq_reindex_queue_scrape_pkey', '0 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_pkey;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_active_locked_at', '20 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_active_locked_at_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_queued_optimal_2', '40 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_queued_optimal_2_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_failed_created_at', '0 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_failed_created_at_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_completed_created_at', '20 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_completed_created_at_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_group_owner_mode', '40 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_owner_mode_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_group_mode_status', '0 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_mode_status_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_group_completed_listing','20 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_completed_listing_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_group_status', '40 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.idx_queue_scrape_group_status;$$);++SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_pkey', '0 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_backlog_pkey;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_owner_id', '20 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_owner_id_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_group_mode', '40 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_group_mode_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_group_id', '0 6 * * *', $$REINDEX INDEX CONCURRENTLY nuq.idx_queue_scrape_backlog_group_id;$$);+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_times_out_at', '20 6 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_times_out_at_idx;$$);++-- Watchdog: cancel any nuq REINDEX CONCURRENTLY that has been running > 18 min.+-- Acts as the safety net since statement_timeout cannot be set inline with+-- REINDEX CONCURRENTLY. Slots are 20 min apart, so the watchdog must run+-- frequently enough that a stuck job is killed before the next slot fires.+-- Cadence (1 min) + threshold (18 min) caps actual reindex runtime at ~19 min,+-- strictly under the 20 min cadence.+SELECT cron.schedule('nuq_maintenance_watchdog', '* * * * *', $$+ SELECT pg_cancel_backend(pid)+ FROM pg_stat_activity+ WHERE backend_type = 'client backend'+ AND query ILIKE 'REINDEX INDEX CONCURRENTLY nuq.%'+ AND now() - query_start > interval '18 minutes';$$);CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (@@ -171,9 +209,13 @@ SELECT cron.schedule('nuq_queue_crawl_finished_lock_reaper', '15 seconds', $$SELECT pg_notify('nuq.queue_crawl_finished', (id::text || '|' || 'failed'::text)) FROM stallfail;$$);-SELECT cron.schedule('nuq_queue_crawl_finished_reindex', '0 9 * * *', $$- REINDEX TABLE CONCURRENTLY nuq.queue_crawl_finished;-$$);+-- Per-index REINDEX CONCURRENTLY for queue_crawl_finished. See note above on+-- why these are single-statement (no inline statement_timeout).+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_pkey', '0 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_crawl_finished_pkey;$$);+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_active_locked_at', '20 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_crawl_finished_active_locked_at_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_queued_optimal_2', '40 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_queued_optimal_2_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_failed_created_at', '0 8 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_failed_created_at_idx;$$);+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_completed_created_at', '20 8 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_completed_created_at_idx;$$);CREATE TABLE IF NOT EXISTS nuq.group_crawl (id uuid NOT NULL,@@ -214,6 +256,7 @@ SELECT cron.schedule('nuq_group_crawl_finished', '15 seconds', $$$$);SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$+ SET statement_timeout = '4min';WITH cleaned_groups AS (DELETE FROM nuq.group_crawlWHERE nuq.group_crawl.status = 'completed'::nuq.group_status@@ -231,3 +274,9 @@ SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$)SELECT 1;$$);++-- pg_cron does not auto-prune cron.job_run_details; with sub-minute crons it+-- grows unbounded and seq-scans get unusable. Keep the last 24h.+SELECT cron.schedule('cron_job_run_details_prune', '0 * * * *', $$+ DELETE FROM cron.job_run_details WHERE start_time < now() - interval '24 hours';+$$);