Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(nuq): add source for max_concurrency (#2293)

Gergő Móricze61d30bf1515
diff --git a/apps/api/package.json b/apps/api/package.json
index 173187409..303b87e8b 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -71,7 +71,6 @@
"@ai-sdk/fireworks": "^0.2.4",
"@ai-sdk/google": "^1.2.3",
"@ai-sdk/google-vertex": "^2.2.15",
- "@pinecone-database/pinecone": "^6.1.2",
"@ai-sdk/groq": "^1.2.1",
"@ai-sdk/openai": "^1.3.12",
"@apidevtools/json-schema-ref-parser": "^11.7.3",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 43bf374bf..d0be91535 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -76,9 +76,6 @@ importers:
'@opentelemetry/sdk-node':
specifier: ^0.205.0
version: 0.205.0(@opentelemetry/[email protected])
- '@pinecone-database/pinecone':
- specifier: ^6.1.2
- version: 6.1.2
'@sentry/cli':
specifier: ^2.33.1
version: 2.33.1([email protected])
@@ -2626,10 +2623,6 @@ packages:
resolution: {integrity: sha512-IHnV6A+zxU7XwmKFinmYjUcwlyK9+xkG3/s9KcQhI9BjQKycrJ1JRO+FbNYPwZiPKW3je/DR0k7w8/gLa5eaxQ==}
deprecated: 'The package is now available as "qr": npm install qr'
- '@pinecone-database/[email protected]':
- resolution: {integrity: sha512-ydIlbtgIIHFgBL08sPzua5ckmOgtjgDz8xg21CnP1fqnnEgDmOlnfd10MRKU+fvFRhDlh4Md37SwZDr0d4cBqg==}
- engines: {node: '>=18.0.0'}
-
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -9802,8 +9795,6 @@ snapshots:
'@paulmillr/[email protected]': {}
- '@pinecone-database/[email protected]': {}
-
optional: true
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index ba9fec277..19d096fed 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -2,6 +2,7 @@ import { configDotenv } from "dotenv";
configDotenv();
import { TeamFlags } from "../../controllers/v1/types";
+import { Client as PgClient } from "pg";
// =========================================
// Configuration
@@ -43,12 +44,14 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {
runNumber = 0;
}
+ const concurrency = req.concurrency ?? 100;
+
const res = await fetch(process.env.IDMUX_URL + "/", {
method: "POST",
body: JSON.stringify({
refName: process.env.GITHUB_REF_NAME!,
runNumber,
- concurrency: req.concurrency ?? 100,
+ concurrency,
...req,
}),
headers: {
@@ -61,7 +64,24 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {
}
expect(res.ok).toBe(true);
- return await res.json();
+ const result: Identity = await res.json();
+
+ const nuqClient = new PgClient({
+ connectionString: process.env.NUQ_DATABASE_URL,
+ });
+
+ // update the concurrency limit for the team in the nuq spoof table
+ await nuqClient.query(
+ `
+ INSERT INTO nuq.queue_scrape_owner_concurrency_source (id, max_concurrency) VALUES ($1, $2)
+ ON CONFLICT (id) DO UPDATE SET max_concurrency = EXCLUDED.max_concurrency
+ `,
+ [result.teamId, concurrency],
+ );
+
+ await nuqClient.end();
+
+ return result;
}
export type Identity = {
diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.ts
index bfee651cf..dd0f1da67 100644
--- a/apps/api/src/lib/search-index-client.ts
+++ b/apps/api/src/lib/search-index-client.ts
@@ -1,6 +1,6 @@
/**
* HTTP Client for Search Index Service
- *
+ *
* This client communicates with the standalone search index service
* (firecrawl-search backend) via HTTP.
*/
@@ -8,13 +8,13 @@
import { logger as _logger } from "./logger";
import type { Logger } from "winston";
-export interface SearchIndexClientConfig {
+interface SearchIndexClientConfig {
baseUrl: string;
apiSecret?: string;
timeout?: number;
}
-export interface IndexDocumentRequest {
+interface IndexDocumentRequest {
url: string;
resolvedUrl: string;
title?: string;
@@ -43,7 +43,7 @@ export interface SearchRequest {
};
}
-export interface SearchResult {
+interface SearchResult {
documentId: string;
url: string;
title: string | null;
@@ -58,7 +58,7 @@ export interface SearchResult {
lastCrawledAt: string;
}
-export interface SearchResponse {
+interface SearchResponse {
results: SearchResult[];
total: number;
query: string;
@@ -75,13 +75,13 @@ export class SearchIndexClient {
private baseUrl: string;
private apiSecret?: string;
private timeout: number;
-
+
constructor(config: SearchIndexClientConfig) {
this.baseUrl = config.baseUrl.replace(/\/$/, ""); // Remove trailing slash
this.apiSecret = config.apiSecret;
this.timeout = config.timeout || 30000;
}
-
+
/**
* Check if search index service is enabled
*/
@@ -91,7 +91,7 @@ export class SearchIndexClient {
process.env.ENABLE_SEARCH_INDEX === "true"
);
}
-
+
/**
* Make HTTP request to search service
*/
@@ -103,36 +103,36 @@ export class SearchIndexClient {
): Promise<T> {
const log = logger ?? _logger.child({ module: "search-index-client" });
const url = `${this.baseUrl}${path}`;
-
+
const headers: Record<string, string> = {
"Content-Type": "application/json",
};
-
+
if (this.apiSecret) {
headers["X-API-Secret"] = this.apiSecret;
}
-
+
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
-
+
try {
log.debug("Making request to search service", {
method,
url,
hasBody: !!body,
});
-
+
const response = await fetch(url, {
method,
headers,
body: body ? JSON.stringify(body) : undefined,
signal: controller.signal,
});
-
+
clearTimeout(timeoutId);
-
+
const data = await response.json();
-
+
if (!response.ok) {
log.error("Search service request failed", {
status: response.status,
@@ -142,16 +142,19 @@ export class SearchIndexClient {
data.error || `Search service returned ${response.status}`,
);
}
-
+
return data;
} catch (error) {
clearTimeout(timeoutId);
-
+
if (error.name === "AbortError") {
- log.error("Search service request timed out", { url, timeout: this.timeout });
+ log.error("Search service request timed out", {
+ url,
+ timeout: this.timeout,
+ });
throw new Error("Search service request timed out");
}
-
+
log.error("Search service request failed", {
error: (error as Error).message,
url,
@@ -159,7 +162,7 @@ export class SearchIndexClient {
throw error;
}
}
-
+
/**
* Index a document (async - queues for processing)
*/
@@ -168,7 +171,7 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<{ success: boolean; message: string }> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<any>(
"POST",
@@ -176,12 +179,12 @@ export class SearchIndexClient {
request,
log,
);
-
+
log.info("Document queued for indexing", {
url: request.url,
success: response.success,
});
-
+
return {
success: response.success,
message: response.message || "Document queued for indexing",
@@ -191,7 +194,7 @@ export class SearchIndexClient {
error: (error as Error).message,
url: request.url,
});
-
+
// Don't throw - indexing failures shouldn't break scraping
return {
success: false,
@@ -199,7 +202,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Search indexed documents
*/
@@ -208,32 +211,30 @@ export class SearchIndexClient {
logger?: Logger,
): Promise<SearchResponse> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
- const response = await this.request<{ success: boolean; data: SearchResponse }>(
- "POST",
- "/api/search",
- request,
- log,
- );
-
+ const response = await this.request<{
+ success: boolean;
+ data: SearchResponse;
+ }>("POST", "/api/search", request, log);
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
log.info("Search completed", {
query: request.query,
results: response.data.results.length,
took: response.data.took,
});
-
+
return response.data;
} catch (error) {
log.error("Search request failed", {
error: (error as Error).message,
query: request.query,
});
-
+
// Return empty results on error
return {
results: [],
@@ -249,7 +250,7 @@ export class SearchIndexClient {
};
}
}
-
+
/**
* Get search index statistics
*/
@@ -258,7 +259,7 @@ export class SearchIndexClient {
queue: any;
}> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean; data: any }>(
"GET",
@@ -266,30 +267,30 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
if (!response.success || !response.data) {
throw new Error("Invalid response from search service");
}
-
+
return response.data;
} catch (error) {
log.error("Failed to get search stats", {
error: (error as Error).message,
});
-
+
return {
index: {},
queue: {},
};
}
}
-
+
/**
* Health check
*/
async health(logger?: Logger): Promise<boolean> {
const log = logger ?? _logger.child({ module: "search-index-client" });
-
+
try {
const response = await this.request<{ success: boolean }>(
"GET",
@@ -297,7 +298,7 @@ export class SearchIndexClient {
undefined,
log,
);
-
+
return response.success;
} catch (error) {
log.warn("Search service health check failed", {
@@ -318,22 +319,22 @@ export function getSearchIndexClient(): SearchIndexClient | null {
if (!SearchIndexClient.isEnabled()) {
return null;
}
-
+
if (!searchIndexClient) {
const baseUrl = process.env.SEARCH_SERVICE_URL!;
const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;
-
+
searchIndexClient = new SearchIndexClient({
baseUrl,
apiSecret,
timeout: 30000,
});
-
+
_logger.info("Search index client initialized", {
baseUrl: baseUrl.substring(0, 30) + "...",
});
}
-
+
return searchIndexClient;
}
@@ -345,11 +346,11 @@ export async function indexDocumentIfEnabled(
logger?: Logger,
): Promise<void> {
const client = getSearchIndexClient();
-
+
if (!client) {
return;
}
-
+
try {
await client.indexDocument(request, logger);
} catch (error) {
@@ -360,4 +361,3 @@ export async function indexDocumentIfEnabled(
});
}
}
-
diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.ts
deleted file mode 100644
index 8b199f323..000000000
--- a/apps/api/src/services/search-index-db.ts
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Search Index Database Service
- *
- * Separate Supabase client for the search index database.
- * This allows independent scaling and isolation from the main index DB.
- */
-
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { logger as _logger } from "../lib/logger";
-import { configDotenv } from "dotenv";
-
-configDotenv();
-
-/**
- * SearchIndexSupabaseService - Manages connection to search index database
- */
-class SearchIndexSupabaseService {
- private client: SupabaseClient | null = null;
-
- constructor() {
- const supabaseUrl = process.env.SEARCH_INDEX_SUPABASE_URL;
- const supabaseServiceToken = process.env.SEARCH_INDEX_SUPABASE_SERVICE_TOKEN;
-
- // Only initialize if both URL and token are provided
- if (!supabaseUrl || !supabaseServiceToken) {
- _logger.warn("Search index database not configured. Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN to enable.");
- this.client = null;
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken);
- _logger.info("Search index database client initialized", {
- url: supabaseUrl.substring(0, 30) + "...",
- });
- }
- }
-
- /**
- * Get the Supabase client for search index
- */
- getClient(): SupabaseClient | null {
- return this.client;
- }
-
- /**
- * Check if search index is enabled
- */
- isEnabled(): boolean {
- return this.client !== null;
- }
-}
-
-const searchIndexService = new SearchIndexSupabaseService();
-
-/**
- * Proxy to provide clean error messages when search index is not configured
- */
-export const search_index_supabase_service: SupabaseClient = new Proxy(
- searchIndexService,
- {
- get: function (target, prop, receiver) {
- const client = target.getClient();
-
- // If client is not initialized, throw immediately to prevent nested access errors
- if (client === null) {
- throw new Error(
- "Search index database is not configured. " +
- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."
- );
- }
-
- // Direct access to service properties
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
-
- // Delegate to Supabase client
- return Reflect.get(client, prop, receiver);
- },
- }
-) as unknown as SupabaseClient;
-
-/**
- * Check if search index database is enabled
- */
-export function isSearchIndexEnabled(): boolean {
- return (
- process.env.ENABLE_SEARCH_INDEX === "true" &&
- searchIndexService.isEnabled()
- );
-}
-
-/**
- * Get search index database client (throws if not configured)
- */
-export function getSearchIndexClient(): SupabaseClient {
- const client = searchIndexService.getClient();
- if (!client) {
- throw new Error(
- "Search index database is not configured. " +
- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN."
- );
- }
- return client;
-}
-
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index e7eece097..af973cdb3 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -37,16 +37,19 @@ const listenChannelId = process.env.NUQ_POD_NAME ?? "main";
type NuQOptions = {
concurrencyLimit?: false | "per-owner";
-}
+};
type NuQJobOptions = {
listenable?: boolean;
priority?: number;
ownerId?: string;
-}
+};
class NuQ<JobData = any, JobReturnValue = any> {
- constructor(public readonly queueName: string, public readonly options: NuQOptions = {}) {}
+ constructor(
+ public readonly queueName: string,
+ public readonly options: NuQOptions = {},
+ ) {}
// === Listener
@@ -96,7 +99,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
let reconnectTimeout: NodeJS.Timeout | null = null;
- const onClose = (function onClose() {
+ const onClose = function onClose() {
logger.info("NuQ listener channel closed", {
module: "nuq/rabbitmq",
});
@@ -115,7 +118,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
250,
);
return;
- }).bind(this);
+ }.bind(this);
connection.on("close", onClose);
channel.on("close", onClose);
@@ -125,7 +128,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
(msg => {
if (msg === null) {
onClose();
- return;
+ return;
}
logger.info("NuQ job received", {
@@ -325,7 +328,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
"returnvalue",
"failedreason",
"lock",
- "owner_id"
+ "owner_id",
];
private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {
@@ -533,7 +536,13 @@ class NuQ<JobData = any, JobReturnValue = any> {
(
await nuqPool.query(
`INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id) VALUES ($1, $2, $3, $4, $5) RETURNING ${this.jobReturning.join(", ")};`,
- [id, data, options.priority ?? 0, options.listenable ? listenChannelId : null, options.ownerId ?? null],
+ [
+ id,
+ data,
+ options.priority ?? 0,
+ options.listenable ? listenChannelId : null,
+ options.ownerId ?? null,
+ ],
)
).rows[0],
)!;
@@ -690,7 +699,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
FROM next
WHERE q.id = next.id
RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}
- `
+ `;
let updateQuery: string;
if (this.options.concurrencyLimit === "per-owner") {
@@ -708,10 +717,21 @@ class NuQ<JobData = any, JobReturnValue = any> {
WHERE owner_id IS NOT NULL
GROUP BY owner_id
),
+ owner_counts_with_max AS (
+ SELECT
+ oc.owner_id,
+ oc.job_count,
+ CASE
+ WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = oc.owner_id)
+ THEN 0
+ ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)
+ END as max_concurrency
+ FROM owner_counts oc
+ ),
owner_increment AS (
INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)
- SELECT owner_id, job_count, ${/* TODO max concurrency */ 100}
- FROM owner_counts
+ SELECT owner_id, job_count, max_concurrency
+ FROM owner_counts_with_max
ON CONFLICT (id)
DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency
)
@@ -805,11 +825,21 @@ class NuQ<JobData = any, JobReturnValue = any> {
updated AS (
${queryUpdateStatus}
),
+ updated_with_max AS (
+ SELECT
+ u.*,
+ CASE
+ WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = u.owner_id)
+ THEN 0
+ ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(u.owner_id)
+ END as max_concurrency
+ FROM updated u
+ WHERE u.owner_id IS NOT NULL
+ ),
owner_increment AS (
INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)
- SELECT owner_id, 1, ${/* TODO max concurrency */ 100}
- FROM updated
- WHERE owner_id IS NOT NULL
+ SELECT owner_id, 1, max_concurrency
+ FROM updated_with_max
ON CONFLICT (id)
DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1
)
@@ -895,7 +925,11 @@ class NuQ<JobData = any, JobReturnValue = any> {
updateQuery = `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;
}
- const result = await nuqPool.query(updateQuery, [id, lock, returnvalue]);
+ const result = await nuqPool.query(updateQuery, [
+ id,
+ lock,
+ returnvalue,
+ ]);
const success = result.rowCount !== 0;
@@ -903,10 +937,14 @@ class NuQ<JobData = any, JobReturnValue = any> {
const job = result.rows[0];
if (job) {
- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
- job.id + "|completed",
- ]);
+ if (
+ this.nuqWaitMode === "listen" &&
+ !process.env.NUQ_RABBITMQ_URL
+ ) {
+ await nuqPool.query(
+ `SELECT pg_notify('${this.queueName}', $1);`,
+ [job.id + "|completed"],
+ );
} else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
await this.sendJobEnd(
job.id,
@@ -975,7 +1013,11 @@ class NuQ<JobData = any, JobReturnValue = any> {
updateQuery = `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;
}
- const result = await nuqPool.query(updateQuery, [id, lock, failedReason]);
+ const result = await nuqPool.query(updateQuery, [
+ id,
+ lock,
+ failedReason,
+ ]);
const success = result.rowCount !== 0;
@@ -983,10 +1025,14 @@ class NuQ<JobData = any, JobReturnValue = any> {
const job = result.rows[0];
if (job) {
- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {
- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [
- job.id + "|failed",
- ]);
+ if (
+ this.nuqWaitMode === "listen" &&
+ !process.env.NUQ_RABBITMQ_URL
+ ) {
+ await nuqPool.query(
+ `SELECT pg_notify('${this.queueName}', $1);`,
+ [job.id + "|failed"],
+ );
} else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {
await this.sendJobEnd(
job.id,
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 074c2d374..a0098f150 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (
CONSTRAINT queue_scrape_owner_concurrency_pkey PRIMARY KEY (id)
);
+-- fake concurrency limit source for tests
+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency_source (
+ id uuid NOT NULL,
+ max_concurrency int8 NOT NULL,
+ CONSTRAINT queue_scrape_owner_concurrency_source_pkey PRIMARY KEY (id)
+);
+
+CREATE OR REPLACE FUNCTION nuq_queue_scrape_owner_resolve_max_concurrency(owner_id uuid)
+RETURNS int8
+LANGUAGE sql
+STABLE
+AS $$
+ SELECT COALESCE((SELECT max_concurrency FROM nuq.queue_scrape_owner_concurrency_source WHERE id = owner_id LIMIT 1), 100)::int8;
+$$;
+
SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$
DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';
$$);
@@ -122,4 +137,7 @@ SELECT cron.schedule('nuq_queue_scrape_concurrency_sync', '*/5 * * * *', $$
WHERE nuq.queue_scrape.owner_id = nuq.queue_scrape_owner_concurrency.id
AND nuq.queue_scrape.status = 'active'::nuq.job_status
);
+
+ UPDATE nuq.queue_scrape_owner_concurrency
+ SET max_concurrency = (SELECT nuq_queue_scrape_owner_resolve_max_concurrency(nuq.queue_scrape_owner_concurrency.id));
$$);