Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(nuq): add owner_id tracking to each row
Gergő Móricz5eb3a99df9a8
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex f09b34c1a..c70e30416 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -346,8 +346,11 @@ export async function concurrentJobDone(job: NuQJob<any>) {...nextJob.job.data,concurrencyLimitHit: true,},- nextJob.job.priority,- nextJob.job.listenable,+ {+ priority: nextJob.job.priority,+ listenable: nextJob.job.listenable,+ ownerId: nextJob.job.data.team_id,+ },);}}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 885be5005..99bd7d0de 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -86,7 +86,11 @@ export async function _addScrapeJobToBullMQ(}}- return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);+ return await scrapeQueue.addJob(jobId, webScraperOptions, {+ priority,+ listenable,+ ownerId: webScraperOptions.team_id,+ });}async function addScrapeJobRaw(diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 13a49a555..a1a5ac968 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -28,14 +28,25 @@ export type NuQJob<Data = any, ReturnValue = any> = {returnvalue?: ReturnValue;failedReason?: string;lock?: string;+ ownerId?: string;};const listenChannelId = process.env.NUQ_POD_NAME ?? "main";// === Queue+type NuQOptions = {+ concurrencyLimit?: false | "per-owner";+}++type NuQJobOptions = {+ listenable?: boolean;+ priority?: number;+ ownerId?: string;+}+class NuQ<JobData = any, JobReturnValue = any> {- constructor(public readonly queueName: string) {}+ constructor(public readonly queueName: string, public readonly options: NuQOptions = {}) {}// === Listener@@ -314,6 +325,7 @@ class NuQ<JobData = any, JobReturnValue = any> {"returnvalue","failedreason","lock",+ "owner_id"];private rowToJob(row: any): NuQJob<JobData, JobReturnValue> | null {@@ -329,6 +341,7 @@ class NuQ<JobData = any, JobReturnValue = any> {returnvalue: row.returnvalue ?? undefined,failedReason: row.failedreason ?? undefined,lock: row.lock ?? undefined,+ ownerId: row.owner_id ?? undefined,};}@@ -503,16 +516,15 @@ class NuQ<JobData = any, JobReturnValue = any> {public async addJob(id: string,data: JobData,- priority: number = 0,- listenable: boolean = false,+ options: NuQJobOptions = {},): Promise<NuQJob<JobData, JobReturnValue>> {return withSpan("nuq.addJob", async span => {setSpanAttributes(span, {"nuq.queue_name": this.queueName,"nuq.job_id": id,- "nuq.priority": priority,+ "nuq.priority": options.priority ?? 0,"nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": listenable,+ "nuq.listenable": options.listenable ?? false,});const start = Date.now();@@ -520,8 +532,8 @@ class NuQ<JobData = any, JobReturnValue = any> {const result = this.rowToJob((await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id) VALUES ($1, $2, $3, $4) RETURNING ${this.jobReturning.join(", ")};`,- [id, data, priority, listenable ? listenChannelId : null],+ `INSERT INTO ${this.queueName} (id, data, priority, listen_channel_id, owner_id) VALUES ($1, $2, $3, $4, $5) RETURNING ${this.jobReturning.join(", ")};`,+ [id, data, options.priority ?? 0, options.listenable ? listenChannelId : null, options.ownerId ?? null],)).rows[0],)!;diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex f22a6b83a..b52c78ec5 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -22,6 +22,7 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape (listen_channel_id text, -- for listenable jobs over rabbitmqreturnvalue jsonb, -- only for selfhostfailedreason text, -- only for selfhost+ owner_id uuid,CONSTRAINT queue_scrape_pkey PRIMARY KEY (id));