Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): preserve backlog status on nuq add (#3842)
Gergő Móricz41ffb6d7ffb6
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 5b30dfe1f..3364dc6c9 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -73,6 +73,12 @@ jobs:- run: pnpm config set store-dir ~/.pnpm-store- run: mkdir -p ~/.pnpm-store+ - name: Install FoundationDB client library+ run: |+ curl -fsSL -o /tmp/fdb-clients.deb \+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"+ sudo dpkg -i /tmp/fdb-clients.deb+- uses: actions/setup-node@v6with:node-version: 22@@ -232,13 +238,6 @@ jobs:docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgresdocker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest- - name: Install FoundationDB client library- if: matrix.nuq == 'fdb'- run: |- curl -fsSL -o /tmp/fdb-clients.deb \- "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"- sudo dpkg -i /tmp/fdb-clients.deb-- name: Start FoundationDBif: matrix.nuq == 'fdb'run: |diff --git a/apps/api/src/__tests__/nuq-postgres.test.ts b/apps/api/src/__tests__/nuq-postgres.test.tsnew file mode 100644index 000000000..46c8204c2--- /dev/null+++ b/apps/api/src/__tests__/nuq-postgres.test.ts@@ -0,0 +1,77 @@+import { randomUUID } from "crypto";+import { Pool } from "pg";+import { config } from "../config";+import { nuqShutdown, scrapeQueue } from "../services/worker/nuq";++const describeIf = config.NUQ_DATABASE_URL ? describe : describe.skip;++describeIf("NuQ Postgres queue", () => {+ let cleanupPool: Pool;+ const ids: string[] = [];++ beforeAll(() => {+ cleanupPool = new Pool({+ connectionString: config.NUQ_DATABASE_URL,+ application_name: "nuq-postgres-test",+ });+ });++ afterEach(async () => {+ if (ids.length === 0) return;+ await cleanupPool.query(+ "DELETE FROM nuq.queue_scrape_backlog WHERE id = ANY($1::uuid[])",+ [ids],+ );+ await cleanupPool.query(+ "DELETE FROM nuq.queue_scrape WHERE id = ANY($1::uuid[])",+ [ids],+ );+ ids.length = 0;+ });++ afterAll(async () => {+ await cleanupPool.end();+ await nuqShutdown();+ });++ function scrapeData() {+ return {+ mode: "single_urls",+ url: "https://example.com",+ team_id: randomUUID(),+ } as any;+ }++ test("single backlogged inserts report backlog status", async () => {+ const addJobId = randomUUID();+ const addJobIfNotExistsId = randomUUID();+ ids.push(addJobId, addJobIfNotExistsId);++ await expect(+ scrapeQueue.addJob(addJobId, scrapeData(), {+ backlogged: true,+ backloggedTimesOutAt: new Date(Date.now() + 60_000),+ }),+ ).resolves.toMatchObject({+ id: addJobId,+ status: "backlog",+ });++ await expect(+ scrapeQueue.addJobIfNotExists(addJobIfNotExistsId, scrapeData(), {+ backlogged: true,+ backloggedTimesOutAt: new Date(Date.now() + 60_000),+ }),+ ).resolves.toMatchObject({+ id: addJobIfNotExistsId,+ status: "backlog",+ });++ await expect(+ scrapeQueue.addJobIfNotExists(addJobIfNotExistsId, scrapeData(), {+ backlogged: true,+ backloggedTimesOutAt: new Date(Date.now() + 60_000),+ }),+ ).resolves.toBeNull();+ });+});diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 627196e20..fb5b1ca64 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -842,6 +842,7 @@ class NuQ<JobData = any, JobReturnValue = any> {],)).rows[0],+ options.backlogged,)!;setSpanAttributes(span, {@@ -902,6 +903,7 @@ class NuQ<JobData = any, JobReturnValue = any> {],)).rows[0],+ options.backlogged,);setSpanAttributes(span, {