Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): add source for max_concurrency (#2293)
Gergő Móricze61d30bf1515
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 173187409..303b87e8b 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -71,7 +71,6 @@"@ai-sdk/fireworks": "^0.2.4","@ai-sdk/google": "^1.2.3","@ai-sdk/google-vertex": "^2.2.15",- "@pinecone-database/pinecone": "^6.1.2","@ai-sdk/groq": "^1.2.1","@ai-sdk/openai": "^1.3.12","@apidevtools/json-schema-ref-parser": "^11.7.3",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 43bf374bf..d0be91535 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -76,9 +76,6 @@ importers:'@opentelemetry/sdk-node':specifier: ^0.205.0version: 0.205.0(@opentelemetry/[email protected])- '@pinecone-database/pinecone':- specifier: ^6.1.2- version: 6.1.2'@sentry/cli':specifier: ^2.33.1version: 2.33.1([email protected])@@ -2626,10 +2623,6 @@ packages:resolution: {integrity: sha512-IHnV6A+zxU7XwmKFinmYjUcwlyK9+xkG3/s9KcQhI9BjQKycrJ1JRO+FbNYPwZiPKW3je/DR0k7w8/gLa5eaxQ==}deprecated: 'The package is now available as "qr": npm install qr'- '@pinecone-database/[email protected]':- resolution: {integrity: sha512-ydIlbtgIIHFgBL08sPzua5ckmOgtjgDz8xg21CnP1fqnnEgDmOlnfd10MRKU+fvFRhDlh4Md37SwZDr0d4cBqg==}- engines: {node: '>=18.0.0'}-'@pkgjs/[email protected]':resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}engines: {node: '>=14'}@@ -9802,8 +9795,6 @@ snapshots:'@paulmillr/[email protected]': {}- '@pinecone-database/[email protected]': {}-'@pkgjs/[email protected]':optional: truediff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex ba9fec277..19d096fed 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -2,6 +2,7 @@ import { configDotenv } from "dotenv";configDotenv();import { TeamFlags } from "../../controllers/v1/types";+import { Client as PgClient } from "pg";// =========================================// Configuration@@ -43,12 +44,14 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {runNumber = 0;}+ const concurrency = req.concurrency ?? 100;+const res = await fetch(process.env.IDMUX_URL + "/", {method: "POST",body: JSON.stringify({refName: process.env.GITHUB_REF_NAME!,runNumber,- concurrency: req.concurrency ?? 100,+ concurrency,...req,}),headers: {@@ -61,7 +64,24 @@ export async function idmux(req: IdmuxRequest): Promise<Identity> {}expect(res.ok).toBe(true);- return await res.json();+ const result: Identity = await res.json();++ const nuqClient = new PgClient({+ connectionString: process.env.NUQ_DATABASE_URL,+ });++ // update the concurrency limit for the team in the nuq spoof table+ await nuqClient.query(+ `+ INSERT INTO nuq.queue_scrape_owner_concurrency_source (id, max_concurrency) VALUES ($1, $2)+ ON CONFLICT (id) DO UPDATE SET max_concurrency = EXCLUDED.max_concurrency+ `,+ [result.teamId, concurrency],+ );++ await nuqClient.end();++ return result;}export type Identity = {diff --git a/apps/api/src/lib/search-index-client.ts b/apps/api/src/lib/search-index-client.tsindex bfee651cf..dd0f1da67 100644--- a/apps/api/src/lib/search-index-client.ts+++ b/apps/api/src/lib/search-index-client.ts@@ -1,6 +1,6 @@/*** HTTP Client for Search Index Service- *+ ** This client communicates with the standalone search index service* (firecrawl-search backend) via HTTP.*/@@ -8,13 +8,13 @@import { logger as _logger } from "./logger";import type { Logger } from "winston";-export interface SearchIndexClientConfig {+interface SearchIndexClientConfig {baseUrl: string;apiSecret?: string;timeout?: number;}-export interface IndexDocumentRequest {+interface IndexDocumentRequest {url: string;resolvedUrl: string;title?: string;@@ -43,7 +43,7 @@ export interface SearchRequest {};}-export interface SearchResult {+interface SearchResult {documentId: string;url: string;title: string | null;@@ -58,7 +58,7 @@ export interface SearchResult {lastCrawledAt: string;}-export interface SearchResponse {+interface SearchResponse {results: SearchResult[];total: number;query: string;@@ -75,13 +75,13 @@ export class SearchIndexClient {private baseUrl: string;private apiSecret?: string;private timeout: number;-+constructor(config: SearchIndexClientConfig) {this.baseUrl = config.baseUrl.replace(/\/$/, ""); // Remove trailing slashthis.apiSecret = config.apiSecret;this.timeout = config.timeout || 30000;}-+/*** Check if search index service is enabled*/@@ -91,7 +91,7 @@ export class SearchIndexClient {process.env.ENABLE_SEARCH_INDEX === "true");}-+/*** Make HTTP request to search service*/@@ -103,36 +103,36 @@ export class SearchIndexClient {): Promise<T> {const log = logger ?? _logger.child({ module: "search-index-client" });const url = `${this.baseUrl}${path}`;-+const headers: Record<string, string> = {"Content-Type": "application/json",};-+if (this.apiSecret) {headers["X-API-Secret"] = this.apiSecret;}-+const controller = new AbortController();const timeoutId = setTimeout(() => controller.abort(), this.timeout);-+try {log.debug("Making request to search service", {method,url,hasBody: !!body,});-+const response = await fetch(url, {method,headers,body: body ? JSON.stringify(body) : undefined,signal: controller.signal,});-+clearTimeout(timeoutId);-+const data = await response.json();-+if (!response.ok) {log.error("Search service request failed", {status: response.status,@@ -142,16 +142,19 @@ export class SearchIndexClient {data.error || `Search service returned ${response.status}`,);}-+return data;} catch (error) {clearTimeout(timeoutId);-+if (error.name === "AbortError") {- log.error("Search service request timed out", { url, timeout: this.timeout });+ log.error("Search service request timed out", {+ url,+ timeout: this.timeout,+ });throw new Error("Search service request timed out");}-+log.error("Search service request failed", {error: (error as Error).message,url,@@ -159,7 +162,7 @@ export class SearchIndexClient {throw error;}}-+/*** Index a document (async - queues for processing)*/@@ -168,7 +171,7 @@ export class SearchIndexClient {logger?: Logger,): Promise<{ success: boolean; message: string }> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<any>("POST",@@ -176,12 +179,12 @@ export class SearchIndexClient {request,log,);-+log.info("Document queued for indexing", {url: request.url,success: response.success,});-+return {success: response.success,message: response.message || "Document queued for indexing",@@ -191,7 +194,7 @@ export class SearchIndexClient {error: (error as Error).message,url: request.url,});-+// Don't throw - indexing failures shouldn't break scrapingreturn {success: false,@@ -199,7 +202,7 @@ export class SearchIndexClient {};}}-+/*** Search indexed documents*/@@ -208,32 +211,30 @@ export class SearchIndexClient {logger?: Logger,): Promise<SearchResponse> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {- const response = await this.request<{ success: boolean; data: SearchResponse }>(- "POST",- "/api/search",- request,- log,- );-+ const response = await this.request<{+ success: boolean;+ data: SearchResponse;+ }>("POST", "/api/search", request, log);+if (!response.success || !response.data) {throw new Error("Invalid response from search service");}-+log.info("Search completed", {query: request.query,results: response.data.results.length,took: response.data.took,});-+return response.data;} catch (error) {log.error("Search request failed", {error: (error as Error).message,query: request.query,});-+// Return empty results on errorreturn {results: [],@@ -249,7 +250,7 @@ export class SearchIndexClient {};}}-+/*** Get search index statistics*/@@ -258,7 +259,7 @@ export class SearchIndexClient {queue: any;}> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<{ success: boolean; data: any }>("GET",@@ -266,30 +267,30 @@ export class SearchIndexClient {undefined,log,);-+if (!response.success || !response.data) {throw new Error("Invalid response from search service");}-+return response.data;} catch (error) {log.error("Failed to get search stats", {error: (error as Error).message,});-+return {index: {},queue: {},};}}-+/*** Health check*/async health(logger?: Logger): Promise<boolean> {const log = logger ?? _logger.child({ module: "search-index-client" });-+try {const response = await this.request<{ success: boolean }>("GET",@@ -297,7 +298,7 @@ export class SearchIndexClient {undefined,log,);-+return response.success;} catch (error) {log.warn("Search service health check failed", {@@ -318,22 +319,22 @@ export function getSearchIndexClient(): SearchIndexClient | null {if (!SearchIndexClient.isEnabled()) {return null;}-+if (!searchIndexClient) {const baseUrl = process.env.SEARCH_SERVICE_URL!;const apiSecret = process.env.SEARCH_SERVICE_API_SECRET;-+searchIndexClient = new SearchIndexClient({baseUrl,apiSecret,timeout: 30000,});-+_logger.info("Search index client initialized", {baseUrl: baseUrl.substring(0, 30) + "...",});}-+return searchIndexClient;}@@ -345,11 +346,11 @@ export async function indexDocumentIfEnabled(logger?: Logger,): Promise<void> {const client = getSearchIndexClient();-+if (!client) {return;}-+try {await client.indexDocument(request, logger);} catch (error) {@@ -360,4 +361,3 @@ export async function indexDocumentIfEnabled(});}}-diff --git a/apps/api/src/services/search-index-db.ts b/apps/api/src/services/search-index-db.tsdeleted file mode 100644index 8b199f323..000000000--- a/apps/api/src/services/search-index-db.ts+++ /dev/null@@ -1,104 +0,0 @@-/**- * Search Index Database Service- *- * Separate Supabase client for the search index database.- * This allows independent scaling and isolation from the main index DB.- */--import { createClient, SupabaseClient } from "@supabase/supabase-js";-import { logger as _logger } from "../lib/logger";-import { configDotenv } from "dotenv";--configDotenv();--/**- * SearchIndexSupabaseService - Manages connection to search index database- */-class SearchIndexSupabaseService {- private client: SupabaseClient | null = null;-- constructor() {- const supabaseUrl = process.env.SEARCH_INDEX_SUPABASE_URL;- const supabaseServiceToken = process.env.SEARCH_INDEX_SUPABASE_SERVICE_TOKEN;-- // Only initialize if both URL and token are provided- if (!supabaseUrl || !supabaseServiceToken) {- _logger.warn("Search index database not configured. Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN to enable.");- this.client = null;- } else {- this.client = createClient(supabaseUrl, supabaseServiceToken);- _logger.info("Search index database client initialized", {- url: supabaseUrl.substring(0, 30) + "...",- });- }- }-- /**- * Get the Supabase client for search index- */- getClient(): SupabaseClient | null {- return this.client;- }-- /**- * Check if search index is enabled- */- isEnabled(): boolean {- return this.client !== null;- }-}--const searchIndexService = new SearchIndexSupabaseService();--/**- * Proxy to provide clean error messages when search index is not configured- */-export const search_index_supabase_service: SupabaseClient = new Proxy(- searchIndexService,- {- get: function (target, prop, receiver) {- const client = target.getClient();-- // If client is not initialized, throw immediately to prevent nested access errors- if (client === null) {- throw new Error(- "Search index database is not configured. " +- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN environment variables."- );- }-- // Direct access to service properties- if (prop in target) {- return Reflect.get(target, prop, receiver);- }-- // Delegate to Supabase client- return Reflect.get(client, prop, receiver);- },- }-) as unknown as SupabaseClient;--/**- * Check if search index database is enabled- */-export function isSearchIndexEnabled(): boolean {- return (- process.env.ENABLE_SEARCH_INDEX === "true" &&- searchIndexService.isEnabled()- );-}--/**- * Get search index database client (throws if not configured)- */-export function getSearchIndexClient(): SupabaseClient {- const client = searchIndexService.getClient();- if (!client) {- throw new Error(- "Search index database is not configured. " +- "Set SEARCH_INDEX_SUPABASE_URL and SEARCH_INDEX_SUPABASE_SERVICE_TOKEN."- );- }- return client;-}-diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex e7eece097..af973cdb3 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -37,16 +37,19 @@ const listenChannelId = process.env.NUQ_POD_NAME ?? "main";type NuQOptions = {concurrencyLimit?: false | "per-owner";-}+};type NuQJobOptions = {listenable?: boolean;priority?: number;ownerId?: string;-}+};class NuQ<JobData = any, JobReturnValue = any> {- constructor(public readonly queueName: string, public readonly options: NuQOptions = {}) {}+ constructor(+ public readonly queueName: string,+ public readonly options: NuQOptions = {},+ ) {}// === Listener@@ -96,7 +99,7 @@ class NuQ<JobData = any, JobReturnValue = any> {let reconnectTimeout: NodeJS.Timeout | null = null;- const onClose = (function onClose() {+ const onClose = function onClose() {logger.info("NuQ listener channel closed", {module: "nuq/rabbitmq",});@@ -115,7 +118,7 @@ class NuQ<JobData = any, JobReturnValue = any> {250,);return;- }).bind(this);+ }.bind(this);connection.on("close", onClose);channel.on("close", onClose);@@ -125,7 +128,7 @@ class NuQ<JobData = any, JobReturnValue = any> {(msg => {if (msg === null) {onClose();- return;+ return;}logger.info("NuQ job received", {@@ -325,7 +328,7 @@ class NuQ<JobData = any, JobReturnValue = any> {"returnvalue","failedreason","lock",- "owner_id"+ "owner_id",];private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {@@ -533,7 +536,13 @@ class NuQ<JobData = any, JobReturnValue = any> {(await nuqPool.query(`INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id) VALUES ($1, $2, $3, $4, $5) RETURNING ${this.jobReturning.join(", ")};`,- [id, data, options.priority ?? 0, options.listenable ? listenChannelId : null, options.ownerId ?? null],+ [+ id,+ data,+ options.priority ?? 0,+ options.listenable ? listenChannelId : null,+ options.ownerId ?? null,+ ],)).rows[0],)!;@@ -690,7 +699,7 @@ class NuQ<JobData = any, JobReturnValue = any> {FROM nextWHERE q.id = next.idRETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}- `+ `;let updateQuery: string;if (this.options.concurrencyLimit === "per-owner") {@@ -708,10 +717,21 @@ class NuQ<JobData = any, JobReturnValue = any> {WHERE owner_id IS NOT NULLGROUP BY owner_id),+ owner_counts_with_max AS (+ SELECT+ oc.owner_id,+ oc.job_count,+ CASE+ WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = oc.owner_id)+ THEN 0+ ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(oc.owner_id)+ END as max_concurrency+ FROM owner_counts oc+ ),owner_increment AS (INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, job_count, ${/* TODO max concurrency */ 100}- FROM owner_counts+ SELECT owner_id, job_count, max_concurrency+ FROM owner_counts_with_maxON CONFLICT (id)DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + EXCLUDED.current_concurrency)@@ -805,11 +825,21 @@ class NuQ<JobData = any, JobReturnValue = any> {updated AS (${queryUpdateStatus}),+ updated_with_max AS (+ SELECT+ u.*,+ CASE+ WHEN EXISTS (SELECT 1 FROM ${this.queueName}_owner_concurrency WHERE id = u.owner_id)+ THEN 0+ ELSE ${this.queueName.replaceAll(".", "_")}_owner_resolve_max_concurrency(u.owner_id)+ END as max_concurrency+ FROM updated u+ WHERE u.owner_id IS NOT NULL+ ),owner_increment AS (INSERT INTO ${this.queueName}_owner_concurrency (id, current_concurrency, max_concurrency)- SELECT owner_id, 1, ${/* TODO max concurrency */ 100}- FROM updated- WHERE owner_id IS NOT NULL+ SELECT owner_id, 1, max_concurrency+ FROM updated_with_maxON CONFLICT (id)DO UPDATE SET current_concurrency = ${this.queueName}_owner_concurrency.current_concurrency + 1)@@ -895,7 +925,11 @@ class NuQ<JobData = any, JobReturnValue = any> {updateQuery = `UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;}- const result = await nuqPool.query(updateQuery, [id, lock, returnvalue]);+ const result = await nuqPool.query(updateQuery, [+ id,+ lock,+ returnvalue,+ ]);const success = result.rowCount !== 0;@@ -903,10 +937,14 @@ class NuQ<JobData = any, JobReturnValue = any> {const job = result.rows[0];if (job) {- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [- job.id + "|completed",- ]);+ if (+ this.nuqWaitMode === "listen" &&+ !process.env.NUQ_RABBITMQ_URL+ ) {+ await nuqPool.query(+ `SELECT pg_notify('${this.queueName}', $1);`,+ [job.id + "|completed"],+ );} else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {await this.sendJobEnd(job.id,@@ -975,7 +1013,11 @@ class NuQ<JobData = any, JobReturnValue = any> {updateQuery = `UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id, listen_channel_id;`;}- const result = await nuqPool.query(updateQuery, [id, lock, failedReason]);+ const result = await nuqPool.query(updateQuery, [+ id,+ lock,+ failedReason,+ ]);const success = result.rowCount !== 0;@@ -983,10 +1025,14 @@ class NuQ<JobData = any, JobReturnValue = any> {const job = result.rows[0];if (job) {- if (this.nuqWaitMode === "listen" && !process.env.NUQ_RABBITMQ_URL) {- await nuqPool.query(`SELECT pg_notify('${this.queueName}', $1);`, [- job.id + "|failed",- ]);+ if (+ this.nuqWaitMode === "listen" &&+ !process.env.NUQ_RABBITMQ_URL+ ) {+ await nuqPool.query(+ `SELECT pg_notify('${this.queueName}', $1);`,+ [job.id + "|failed"],+ );} else if (process.env.NUQ_RABBITMQ_URL && job.listen_channel_id) {await this.sendJobEnd(job.id,diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 074c2d374..a0098f150 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency (CONSTRAINT queue_scrape_owner_concurrency_pkey PRIMARY KEY (id));+-- fake concurrency limit source for tests+CREATE TABLE IF NOT EXISTS nuq.queue_scrape_owner_concurrency_source (+ id uuid NOT NULL,+ max_concurrency int8 NOT NULL,+ CONSTRAINT queue_scrape_owner_concurrency_source_pkey PRIMARY KEY (id)+);++CREATE OR REPLACE FUNCTION nuq_queue_scrape_owner_resolve_max_concurrency(owner_id uuid)+RETURNS int8+LANGUAGE sql+STABLE+AS $$+ SELECT COALESCE((SELECT max_concurrency FROM nuq.queue_scrape_owner_concurrency_source WHERE id = owner_id LIMIT 1), 100)::int8;+$$;+SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';$$);@@ -122,4 +137,7 @@ SELECT cron.schedule('nuq_queue_scrape_concurrency_sync', '*/5 * * * *', $$WHERE nuq.queue_scrape.owner_id = nuq.queue_scrape_owner_concurrency.idAND nuq.queue_scrape.status = 'active'::nuq.job_status);++ UPDATE nuq.queue_scrape_owner_concurrency+ SET max_concurrency = (SELECT nuq_queue_scrape_owner_resolve_max_concurrency(nuq.queue_scrape_owner_concurrency.id));$$);