Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq-postgres): spread reindexes, bound maintenance with statement_timeout (#3467)

firecrawl-spring[bot]635a836c1040
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 1e8fb60e9..742a336f2 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -105,6 +105,9 @@ CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_owner_id_idx ON nuq.queue_sc
-- For getGroupNumericStats backlog query: query by group_id and data->>'mode' on backlog table
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_group_mode_idx ON nuq.queue_scrape_backlog (group_id) WHERE ((data->>'mode') = 'single_urls');
+-- For nuq_queue_scrape_backlog_reaper: avoid seq scan over the entire backlog every minute
+CREATE INDEX IF NOT EXISTS nuq_queue_scrape_backlog_times_out_at_idx ON nuq.queue_scrape_backlog (times_out_at);
+
SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$
DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour' AND group_id IS NULL;
$$);
@@ -120,12 +123,47 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
$$);
SELECT cron.schedule('nuq_queue_scrape_backlog_reaper', '* * * * *', $$
+ SET statement_timeout = '50s';
DELETE FROM nuq.queue_scrape_backlog
WHERE nuq.queue_scrape_backlog.times_out_at < now();
$$);
-SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$
- REINDEX TABLE CONCURRENTLY nuq.queue_scrape;
+-- Per-index REINDEX CONCURRENTLY, spread across the global low-traffic window
+-- (02:00-06:20 UTC). REINDEX INDEX takes a ShareUpdateExclusiveLock only on
+-- the target index, not the whole table.
+--
+-- Note: REINDEX CONCURRENTLY cannot run inside a transaction block, and pg_cron
+-- wraps multi-statement commands in an implicit transaction. So each cron body
+-- must be a single REINDEX statement -- statement_timeout cannot be set
+-- inline. Stuck reindexes are caught by nuq_maintenance_watchdog below.
+SELECT cron.schedule('nuq_reindex_queue_scrape_pkey', '0 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_pkey;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_active_locked_at', '20 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_active_locked_at_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_queued_optimal_2', '40 2 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_queued_optimal_2_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_failed_created_at', '0 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_failed_created_at_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_completed_created_at', '20 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_completed_created_at_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_group_owner_mode', '40 3 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_owner_mode_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_group_mode_status', '0 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_mode_status_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_group_completed_listing','20 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_group_completed_listing_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_group_status', '40 4 * * *', $$REINDEX INDEX CONCURRENTLY nuq.idx_queue_scrape_group_status;$$);
+
+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_pkey', '0 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_scrape_backlog_pkey;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_owner_id', '20 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_owner_id_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_group_mode', '40 5 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_group_mode_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_group_id', '0 6 * * *', $$REINDEX INDEX CONCURRENTLY nuq.idx_queue_scrape_backlog_group_id;$$);
+SELECT cron.schedule('nuq_reindex_queue_scrape_backlog_times_out_at', '20 6 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_scrape_backlog_times_out_at_idx;$$);
+
+-- Watchdog: cancel any nuq REINDEX CONCURRENTLY that has been running > 18 min.
+-- Acts as the safety net since statement_timeout cannot be set inline with
+-- REINDEX CONCURRENTLY. Slots are 20 min apart, so the watchdog must run
+-- frequently enough that a stuck job is killed before the next slot fires.
+-- Cadence (1 min) + threshold (18 min) caps actual reindex runtime at ~19 min,
+-- strictly under the 20 min cadence.
+SELECT cron.schedule('nuq_maintenance_watchdog', '* * * * *', $$
+ SELECT pg_cancel_backend(pid)
+ FROM pg_stat_activity
+ WHERE backend_type = 'client backend'
+ AND query ILIKE 'REINDEX INDEX CONCURRENTLY nuq.%'
+ AND now() - query_start > interval '18 minutes';
$$);
CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (
@@ -171,9 +209,13 @@ SELECT cron.schedule('nuq_queue_crawl_finished_lock_reaper', '15 seconds', $$
SELECT pg_notify('nuq.queue_crawl_finished', (id::text || '|' || 'failed'::text)) FROM stallfail;
$$);
-SELECT cron.schedule('nuq_queue_crawl_finished_reindex', '0 9 * * *', $$
- REINDEX TABLE CONCURRENTLY nuq.queue_crawl_finished;
-$$);
+-- Per-index REINDEX CONCURRENTLY for queue_crawl_finished. See note above on
+-- why these are single-statement (no inline statement_timeout).
+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_pkey', '0 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_crawl_finished_pkey;$$);
+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_active_locked_at', '20 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.queue_crawl_finished_active_locked_at_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_queued_optimal_2', '40 7 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_queued_optimal_2_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_failed_created_at', '0 8 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_failed_created_at_idx;$$);
+SELECT cron.schedule('nuq_reindex_queue_crawl_finished_completed_created_at', '20 8 * * *', $$REINDEX INDEX CONCURRENTLY nuq.nuq_queue_crawl_finished_completed_created_at_idx;$$);
CREATE TABLE IF NOT EXISTS nuq.group_crawl (
id uuid NOT NULL,
@@ -214,6 +256,7 @@ SELECT cron.schedule('nuq_group_crawl_finished', '15 seconds', $$
$$);
SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$
+ SET statement_timeout = '4min';
WITH cleaned_groups AS (
DELETE FROM nuq.group_crawl
WHERE nuq.group_crawl.status = 'completed'::nuq.group_status
@@ -231,3 +274,9 @@ SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$
)
SELECT 1;
$$);
+
+-- pg_cron does not auto-prune cron.job_run_details; with sub-minute crons it
+-- grows unbounded and seq-scans get unusable. Keep the last 24h.
+SELECT cron.schedule('cron_job_run_details_prune', '0 * * * *', $$
+ DELETE FROM cron.job_run_details WHERE start_time < now() - interval '24 hours';
+$$);