Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { Queue } from "bullmq";
import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
import { addExtractJob, ExtractJobData } from "./extract-queue";
import type { BillTeamJobData } from "./billing/types";
import type { TrackParams } from "./autumn/types";
let loggingQueue: Queue;
let indexQueue: Queue;
let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue<BillTeamJobData>;
let precrawlQueue: Queue;
let firebillTrackRetryQueue: Queue<FirebillTrackRetryJobData>;
let redisConnection: IORedis;
export function getRedisConnection(): IORedis {
if (!redisConnection) {
redisConnection = new IORedis(config.REDIS_URL!, {
maxRetriesPerRequest: null,
});
redisConnection.on("connect", () => logger.info("Redis connected"));
redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
redisConnection.on("error", err => logger.warn("Redis error", { err }));
}
return redisConnection;
}
const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
const deepResearchQueueName = "{deepResearchQueue}";
const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
const firebillTrackRetryQueueName = "{firebillTrackRetryQueue}";
/** A usage event firebill never confirmed, re-sent as the identical event. */
export type FirebillTrackRetryJobData = TrackParams & {
idempotencyKey: string;
};
export async function addExtractJobToQueue(
extractId: string,
data: ExtractJobData,
): Promise<void> {
await addExtractJob(extractId, data);
}
export function getGenerateLlmsTxtQueue() {
if (!generateLlmsTxtQueue) {
generateLlmsTxtQueue = new Queue(generateLlmsTxtQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 90000, // 25 hours
},
removeOnFail: {
age: 90000, // 25 hours
},
},
});
}
return generateLlmsTxtQueue;
}
export function getDeepResearchQueue() {
if (!deepResearchQueue) {
deepResearchQueue = new Queue<DeepResearchServiceOptions>(
deepResearchQueueName,
{
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 90000, // 25 hours
},
removeOnFail: {
age: 90000, // 25 hours
},
},
},
);
}
return deepResearchQueue;
}
export function getBillingQueue() {
if (!billingQueue) {
billingQueue = new Queue<BillTeamJobData>(billingQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 60, // 1 minute
},
removeOnFail: {
age: 3600, // 1 hour
},
},
});
}
return billingQueue;
}
export function getFirebillTrackRetryQueue() {
if (!firebillTrackRetryQueue) {
firebillTrackRetryQueue = new Queue<FirebillTrackRetryJobData>(
firebillTrackRetryQueueName,
{
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 60 * 60, // 1 hour
},
// Exhausted retries stay inspectable for a week: each one is an
// event that may not have been billed.
removeOnFail: {
age: 7 * 24 * 60 * 60, // 7 days
},
},
},
);
}
return firebillTrackRetryQueue;
}
export function getPrecrawlQueue() {
if (!precrawlQueue) {
precrawlQueue = new Queue(precrawlQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 24 * 60 * 60, // 1 day
},
removeOnFail: {
age: 24 * 60 * 60, // 1 day
},
},
});
}
return precrawlQueue;
}