Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq-postgres): tune checkpoint and autovacuum to reduce job prefetch stalls (#2673)

firecrawl-spring[bot]de0d08493984
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 0b78d4e61..4195aa199 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -1,6 +1,14 @@
CREATE EXTENSION IF NOT EXISTS pgcrypto;
CREATE EXTENSION IF NOT EXISTS pg_cron;
+-- Checkpoint tuning: spread I/O to reduce stalls during heavy WAL activity
+-- These settings help prevent prefetch queries from returning 0 jobs during checkpoints
+ALTER SYSTEM SET checkpoint_completion_target = 0.95;
+ALTER SYSTEM SET max_wal_size = '4GB';
+ALTER SYSTEM SET bgwriter_lru_maxpages = 500;
+ALTER SYSTEM SET bgwriter_delay = '100ms';
+SELECT pg_reload_conf();
+
CREATE SCHEMA IF NOT EXISTS nuq;
DO $$ BEGIN
@@ -36,8 +44,8 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (
ALTER TABLE nuq.queue_scrape
SET (autovacuum_vacuum_scale_factor = 0.01,
autovacuum_analyze_scale_factor = 0.01,
- autovacuum_vacuum_cost_limit = 2000,
- autovacuum_vacuum_cost_delay = 2);
+ autovacuum_vacuum_cost_limit = 10000,
+ autovacuum_vacuum_cost_delay = 0);
CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);
CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);
@@ -116,8 +124,8 @@ CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (
ALTER TABLE nuq.queue_crawl_finished
SET (autovacuum_vacuum_scale_factor = 0.01,
autovacuum_analyze_scale_factor = 0.01,
- autovacuum_vacuum_cost_limit = 2000,
- autovacuum_vacuum_cost_delay = 2);
+ autovacuum_vacuum_cost_limit = 10000,
+ autovacuum_vacuum_cost_delay = 0);
CREATE INDEX IF NOT EXISTS queue_crawl_finished_active_locked_at_idx ON nuq.queue_crawl_finished USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);
CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_queued_optimal_2_idx ON nuq.queue_crawl_finished (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);