Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(nuq): respect advisory lock in lock reaper cron

Gergő Móricz93d1e15f8b1a
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 287dc5d67..5e60c29cd 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -96,45 +96,107 @@ SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$
$$);
SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
- WITH requeued AS (
+ WITH stalled_jobs AS (
+ SELECT id, owner_id, group_id, stalls
+ FROM nuq.queue_scrape
+ WHERE locked_at <= now() - interval '1 minute'
+ AND status = 'active'::nuq.job_status
+ ),
+ distinct_owners AS (
+ SELECT DISTINCT owner_id
+ FROM stalled_jobs
+ WHERE owner_id IS NOT NULL
+ ORDER BY owner_id
+ ),
+ acquired_owner_locks AS (
+ SELECT
+ owner_id,
+ pg_advisory_xact_lock(hashtext(owner_id::text)) as dummy
+ FROM distinct_owners
+ ),
+ distinct_groups AS (
+ SELECT DISTINCT group_id
+ FROM stalled_jobs
+ WHERE group_id IS NOT NULL
+ ORDER BY group_id
+ ),
+ acquired_group_locks AS (
+ SELECT
+ group_id,
+ pg_advisory_xact_lock(hashtext(group_id::text)) as dummy
+ FROM distinct_groups
+ ),
+ requeued AS (
UPDATE nuq.queue_scrape
SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1
- WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute'
- AND nuq.queue_scrape.status = 'active'::nuq.job_status
- AND COALESCE(nuq.queue_scrape.stalls, 0) < 9
- RETURNING id, owner_id
+ WHERE id IN (
+ SELECT sj.id
+ FROM stalled_jobs sj
+ WHERE COALESCE(sj.stalls, 0) < 9
+ AND (sj.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = sj.owner_id))
+ AND (sj.group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = sj.group_id))
+ )
+ RETURNING id, owner_id, group_id
),
- requeued_counts AS (
+ requeued_owner_counts AS (
SELECT owner_id, COUNT(*) as job_count
FROM requeued
WHERE owner_id IS NOT NULL
GROUP BY owner_id
),
- requeue_concurrency_update AS (
+ requeue_owner_concurrency_update AS (
UPDATE nuq.queue_scrape_owner_concurrency
- SET current_concurrency = GREATEST(0, current_concurrency - requeued_counts.job_count)
- FROM requeued_counts
- WHERE nuq.queue_scrape_owner_concurrency.id = requeued_counts.owner_id
+ SET current_concurrency = GREATEST(0, current_concurrency - requeued_owner_counts.job_count)
+ FROM requeued_owner_counts
+ WHERE nuq.queue_scrape_owner_concurrency.id = requeued_owner_counts.owner_id
+ ),
+ requeued_group_counts AS (
+ SELECT group_id, COUNT(*) as job_count
+ FROM requeued
+ WHERE group_id IS NOT NULL
+ GROUP BY group_id
+ ),
+ requeue_group_concurrency_update AS (
+ UPDATE nuq.queue_scrape_group_concurrency
+ SET current_concurrency = GREATEST(0, current_concurrency - requeued_group_counts.job_count)
+ FROM requeued_group_counts
+ WHERE nuq.queue_scrape_group_concurrency.id = requeued_group_counts.group_id
),
stallfail AS (
UPDATE nuq.queue_scrape
SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1
- WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute'
- AND nuq.queue_scrape.status = 'active'::nuq.job_status
- AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9
- RETURNING id, owner_id
+ WHERE id IN (
+ SELECT sj.id
+ FROM stalled_jobs sj
+ WHERE COALESCE(sj.stalls, 0) >= 9
+ AND (sj.owner_id IS NULL OR EXISTS (SELECT 1 FROM acquired_owner_locks WHERE owner_id = sj.owner_id))
+ AND (sj.group_id IS NULL OR EXISTS (SELECT 1 FROM acquired_group_locks WHERE group_id = sj.group_id))
+ )
+ RETURNING id, owner_id, group_id
),
- stallfail_counts AS (
+ stallfail_owner_counts AS (
SELECT owner_id, COUNT(*) as job_count
FROM stallfail
WHERE owner_id IS NOT NULL
GROUP BY owner_id
),
- stallfail_concurrency_update AS (
+ stallfail_owner_concurrency_update AS (
UPDATE nuq.queue_scrape_owner_concurrency
- SET current_concurrency = GREATEST(0, current_concurrency - stallfail_counts.job_count)
- FROM stallfail_counts
- WHERE nuq.queue_scrape_owner_concurrency.id = stallfail_counts.owner_id
+ SET current_concurrency = GREATEST(0, current_concurrency - stallfail_owner_counts.job_count)
+ FROM stallfail_owner_counts
+ WHERE nuq.queue_scrape_owner_concurrency.id = stallfail_owner_counts.owner_id
+ ),
+ stallfail_group_counts AS (
+ SELECT group_id, COUNT(*) as job_count
+ FROM stallfail
+ WHERE group_id IS NOT NULL
+ GROUP BY group_id
+ ),
+ stallfail_group_concurrency_update AS (
+ UPDATE nuq.queue_scrape_group_concurrency
+ SET current_concurrency = GREATEST(0, current_concurrency - stallfail_group_counts.job_count)
+ FROM stallfail_group_counts
+ WHERE nuq.queue_scrape_group_concurrency.id = stallfail_group_counts.group_id
)
SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;
$$);