Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: formatting
Nicolas3b6edef9fa8f
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex f1fe3431b..1ea289959 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -60,7 +60,11 @@ export async function scrapeController(try {doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this} catch (e) {- logger.error(`Error in scrapeController: ${e}`, { jobId, scrapeId: jobId, startTime });+ logger.error(`Error in scrapeController: ${e}`, {+ jobId,+ scrapeId: jobId,+ startTime,+ });if (e instanceof Error &&(e.message.startsWith("Job wait") || e.message === "timeout")diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 602d13b32..6ecb0b8f3 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -94,9 +94,13 @@ export async function addCrawlJobDone(await redisConnection.rpush("crawl:" + id + ":jobs_done_ordered", job_id);} else {// in case it's already been pushed, make sure it's removed- await redisConnection.lrem("crawl:" + id + ":jobs_done_ordered", -1, job_id);+ await redisConnection.lrem(+ "crawl:" + id + ":jobs_done_ordered",+ -1,+ job_id,+ );}-+await redisConnection.expire("crawl:" + id + ":jobs_done_ordered",24 * 60 * 60,diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.tsindex 8d1b95c92..34a5a215e 100644--- a/apps/api/src/lib/extract/completions.ts+++ b/apps/api/src/lib/extract/completions.ts@@ -122,4 +122,3 @@// },// };// }-diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 168d9b8f8..a0c8eaba2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -5,7 +5,7 @@ import { specialtyScrapeCheck } from "../utils/specialtyHandler";export async function scrapeURLWithFetch(meta: Meta,- timeToRun: number | undefined+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {const timeout = timeToRun ?? 300000;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 2dc134c91..14abf9a98 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -128,7 +128,7 @@ export async function scrapeURLWithFireEngineChromeCDP((a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),0,);-+const timeout = (timeToRun ?? 300000) + totalWait;const request: FireEngineScrapeRequestCommon &diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 14f263f30..bb0c485c0 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -105,7 +105,10 @@ export type EngineScrapeResult = {};const engineHandlers: {- [E in Engine]: (meta: Meta, timeToRun: number | undefined) => Promise<EngineScrapeResult>;+ [E in Engine]: (+ meta: Meta,+ timeToRun: number | undefined,+ ) => Promise<EngineScrapeResult>;} = {cache: scrapeCache,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -372,7 +375,7 @@ export function buildFallbackList(meta: Meta): {export async function scrapeURLWithEngine(meta: Meta,engine: Engine,- timeToRun: number | undefined+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {const fn = engineHandlers[engine];const logger = meta.logger.child({diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 0983e4b16..9d2f11b19 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -124,7 +124,10 @@ async function scrapePDFWithParsePDF(};}-export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {+export async function scrapePDF(+ meta: Meta,+ timeToRun: number | undefined,+): Promise<EngineScrapeResult> {if (!meta.options.parsePDF) {const file = await fetchFileToBuffer(meta.url);const content = file.buffer.toString("base64");@@ -152,9 +155,12 @@ export async function scrapePDF(meta: Meta, timeToRun: number | undefined): PromtempFilePath,);-// If the parsed text is under 500 characters and LLAMAPARSE_API_KEY exists, try LlamaParse- if (result.markdown && result.markdown.length < 500 && process.env.LLAMAPARSE_API_KEY) {+ if (+ result.markdown &&+ result.markdown.length < 500 &&+ process.env.LLAMAPARSE_API_KEY+ ) {try {const llamaResult = await scrapePDFWithLlamaParse({@@ -193,4 +199,4 @@ export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Promhtml: result.html,markdown: result.markdown,};-}\ No newline at end of file+}diff --git a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.tsindex db702a44f..38c43878d 100644--- a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts@@ -10,7 +10,10 @@ const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);export function scrapeURLWithScrapingBee(wait_browser: "domcontentloaded" | "networkidle2",): (meta: Meta, timeToRun: number | undefined) => Promise<EngineScrapeResult> {- return async (meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> => {+ return async (+ meta: Meta,+ timeToRun: number | undefined,+ ): Promise<EngineScrapeResult> => {let response: AxiosResponse<any>;const timeout = (timeToRun ?? 300000) + meta.options.waitFor;try {diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 0a4f6e5b6..689f90c80 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -60,9 +60,7 @@ export class SiteError extends Error {export class ActionError extends Error {public code: string;constructor(code: string) {- super(- "Action(s) failed to complete. Error code: " + code,- );+ super("Action(s) failed to complete. Error code: " + code);this.code = code;}}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 93bdb71b3..1df812bdb 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -203,9 +203,10 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const results: EngineResultsTracker = {};let result: EngineScrapeResultWithContext | null = null;- const timeToRun = meta.options.timeout !== undefined- ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))- : undefined+ const timeToRun =+ meta.options.timeout !== undefined+ ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))+ : undefined;for (const { engine, unsupportedFeatures } of fallbackList) {const startedAt = Date.now();diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 6ce48a810..654f6cda0 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -72,7 +72,12 @@ async function addScrapeJobRaw(}if (concurrencyLimited) {- await _addScrapeJobToConcurrencyQueue(webScraperOptions, options, jobId, jobPriority);+ await _addScrapeJobToConcurrencyQueue(+ webScraperOptions,+ options,+ jobId,+ jobPriority,+ );} else {await _addScrapeJobToBullMQ(webScraperOptions, options, jobId, jobPriority);}@@ -130,17 +135,17 @@ export async function addScrapeJobs(let countCanBeDirectlyAdded = Infinity;- if (- jobs[0].data &&- jobs[0].data.team_id &&- jobs[0].data.plan- ) {+ if (jobs[0].data && jobs[0].data.team_id && jobs[0].data.plan) {const now = Date.now();const limit = await getConcurrencyLimitMax(jobs[0].data.plan);console.log("CC limit", limit);cleanOldConcurrencyLimitEntries(jobs[0].data.team_id, now);- countCanBeDirectlyAdded = Math.max(limit - (await getConcurrencyLimitActiveJobs(jobs[0].data.team_id, now)).length, 0);+ countCanBeDirectlyAdded = Math.max(+ limit -+ (await getConcurrencyLimitActiveJobs(jobs[0].data.team_id, now)).length,+ 0,+ );}const addToBull = jobs.slice(0, countCanBeDirectlyAdded);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c2d2e2c64..705a06c78 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -496,15 +496,14 @@ async function processJob(job: Job & { id: string }, token: string) {// See lockURLconst x = await redisConnection.sadd("crawl:" + job.data.crawl_id + ":visited",- ...p1.map(x => x.href),+ ...p1.map((x) => x.href),);const lockRes = x === p1.length;-+if (job.data.crawlerOptions !== null && !lockRes) {throw new RacedRedirectError();}}-}logger.debug("Logging job to DB...");@@ -675,7 +674,10 @@ async function processJob(job: Job & { id: string }, token: string) {logger.debug("Declaring job as done...");await addCrawlJobDone(job.data.crawl_id, job.id, false);- await redisConnection.srem("crawl:" + job.data.crawl_id + ":visited_unique", normalizeURL(job.data.url, sc));+ await redisConnection.srem(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ normalizeURL(job.data.url, sc),+ );logger.debug("Logging job to DB...");await logJob(