Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): preserve backlog status on nuq add (#3842)

Gergő Móricz41ffb6d7ffb6
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 5b30dfe1f..3364dc6c9 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -73,6 +73,12 @@ jobs:
- run: pnpm config set store-dir ~/.pnpm-store
- run: mkdir -p ~/.pnpm-store
+ - name: Install FoundationDB client library
+ run: |
+ curl -fsSL -o /tmp/fdb-clients.deb \
+ "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
+ sudo dpkg -i /tmp/fdb-clients.deb
+
- uses: actions/setup-node@v6
with:
node-version: 22
@@ -232,13 +238,6 @@ jobs:
docker build -t firecrawl/nuq-postgres:latest ./apps/nuq-postgres
docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres --name postgres firecrawl/nuq-postgres:latest
- - name: Install FoundationDB client library
- if: matrix.nuq == 'fdb'
- run: |
- curl -fsSL -o /tmp/fdb-clients.deb \
- "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
- sudo dpkg -i /tmp/fdb-clients.deb
-
- name: Start FoundationDB
if: matrix.nuq == 'fdb'
run: |
diff --git a/apps/api/src/__tests__/nuq-postgres.test.ts b/apps/api/src/__tests__/nuq-postgres.test.ts
new file mode 100644
index 000000000..46c8204c2
--- /dev/null
+++ b/apps/api/src/__tests__/nuq-postgres.test.ts
@@ -0,0 +1,77 @@
+import { randomUUID } from "crypto";
+import { Pool } from "pg";
+import { config } from "../config";
+import { nuqShutdown, scrapeQueue } from "../services/worker/nuq";
+
+const describeIf = config.NUQ_DATABASE_URL ? describe : describe.skip;
+
+describeIf("NuQ Postgres queue", () => {
+ let cleanupPool: Pool;
+ const ids: string[] = [];
+
+ beforeAll(() => {
+ cleanupPool = new Pool({
+ connectionString: config.NUQ_DATABASE_URL,
+ application_name: "nuq-postgres-test",
+ });
+ });
+
+ afterEach(async () => {
+ if (ids.length === 0) return;
+ await cleanupPool.query(
+ "DELETE FROM nuq.queue_scrape_backlog WHERE id = ANY($1::uuid[])",
+ [ids],
+ );
+ await cleanupPool.query(
+ "DELETE FROM nuq.queue_scrape WHERE id = ANY($1::uuid[])",
+ [ids],
+ );
+ ids.length = 0;
+ });
+
+ afterAll(async () => {
+ await cleanupPool.end();
+ await nuqShutdown();
+ });
+
+ function scrapeData() {
+ return {
+ mode: "single_urls",
+ url: "https://example.com",
+ team_id: randomUUID(),
+ } as any;
+ }
+
+ test("single backlogged inserts report backlog status", async () => {
+ const addJobId = randomUUID();
+ const addJobIfNotExistsId = randomUUID();
+ ids.push(addJobId, addJobIfNotExistsId);
+
+ await expect(
+ scrapeQueue.addJob(addJobId, scrapeData(), {
+ backlogged: true,
+ backloggedTimesOutAt: new Date(Date.now() + 60_000),
+ }),
+ ).resolves.toMatchObject({
+ id: addJobId,
+ status: "backlog",
+ });
+
+ await expect(
+ scrapeQueue.addJobIfNotExists(addJobIfNotExistsId, scrapeData(), {
+ backlogged: true,
+ backloggedTimesOutAt: new Date(Date.now() + 60_000),
+ }),
+ ).resolves.toMatchObject({
+ id: addJobIfNotExistsId,
+ status: "backlog",
+ });
+
+ await expect(
+ scrapeQueue.addJobIfNotExists(addJobIfNotExistsId, scrapeData(), {
+ backlogged: true,
+ backloggedTimesOutAt: new Date(Date.now() + 60_000),
+ }),
+ ).resolves.toBeNull();
+ });
+});
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 627196e20..fb5b1ca64 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -842,6 +842,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
],
)
).rows[0],
+ options.backlogged,
)!;
setSpanAttributes(span, {
@@ -902,6 +903,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
],
)
).rows[0],
+ options.backlogged,
);
setSpanAttributes(span, {