Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(nuq-postgres): tune checkpoint and autovacuum to reduce job prefetch stalls (#2673)
firecrawl-spring[bot]de0d08493984
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 0b78d4e61..4195aa199 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -1,6 +1,14 @@CREATE EXTENSION IF NOT EXISTS pgcrypto;CREATE EXTENSION IF NOT EXISTS pg_cron;+-- Checkpoint tuning: spread I/O to reduce stalls during heavy WAL activity+-- These settings help prevent prefetch queries from returning 0 jobs during checkpoints+ALTER SYSTEM SET checkpoint_completion_target = 0.95;+ALTER SYSTEM SET max_wal_size = '4GB';+ALTER SYSTEM SET bgwriter_lru_maxpages = 500;+ALTER SYSTEM SET bgwriter_delay = '100ms';+SELECT pg_reload_conf();+CREATE SCHEMA IF NOT EXISTS nuq;DO $$ BEGIN@@ -36,8 +44,8 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (ALTER TABLE nuq.queue_scrapeSET (autovacuum_vacuum_scale_factor = 0.01,autovacuum_analyze_scale_factor = 0.01,- autovacuum_vacuum_cost_limit = 2000,- autovacuum_vacuum_cost_delay = 2);+ autovacuum_vacuum_cost_limit = 10000,+ autovacuum_vacuum_cost_delay = 0);CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);@@ -116,8 +124,8 @@ CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (ALTER TABLE nuq.queue_crawl_finishedSET (autovacuum_vacuum_scale_factor = 0.01,autovacuum_analyze_scale_factor = 0.01,- autovacuum_vacuum_cost_limit = 2000,- autovacuum_vacuum_cost_delay = 2);+ autovacuum_vacuum_cost_limit = 10000,+ autovacuum_vacuum_cost_delay = 0);CREATE INDEX IF NOT EXISTS queue_crawl_finished_active_locked_at_idx ON nuq.queue_crawl_finished USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_queued_optimal_2_idx ON nuq.queue_crawl_finished (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);