Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
revert all haphazard changes made while bughunting
Gergő Móricz86086ee52816
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 28b1bc5c9..337409d55 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -82,7 +82,7 @@"axios-retry": "^4.5.0","body-parser": "^1.20.3","bottleneck": "^2.19.5",- "bullmq": "^5.56.7",+ "bullmq": "^5.36.0","cacheable-lookup": "^6.1.0","cheerio": "^1.0.0-rc.12","cohere": "^1.1.1",@@ -100,7 +100,7 @@"glob": "^10.4.2","gpt3-tokenizer": "^1.1.5","http-cookie-agent": "^7.0.1",- "ioredis": "^5.6.1",+ "ioredis": "^5.4.1","ip-address": "^10.0.1","joplin-turndown-plugin-gfm": "^1.0.12","jsdom": "^26.0.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 82854b6c8..db4b618f7 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -102,8 +102,8 @@ importers:specifier: ^2.19.5version: 2.19.5bullmq:- specifier: ^5.56.7- version: 5.56.7+ specifier: ^5.36.0+ version: 5.36.0cacheable-lookup:specifier: ^6.1.0version: 6.1.0@@ -156,8 +156,8 @@ importers:specifier: ^7.0.1version: 7.0.1([email protected])([email protected])ioredis:- specifier: ^5.6.1- version: 5.6.1+ specifier: ^5.4.1+ version: 5.4.1ip-address:specifier: ^10.0.1version: 10.0.1@@ -2016,8 +2016,8 @@ packages:resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}- resolution: {integrity: sha512-Aa4Y7rmkuZOuyHvitGd2rSETgQ0SMawPD0XVsU9gAUg9Q4bwNpSD9ZDVcWGhfAQAUZFJH82JUmLsm0B59Prmtg==}+ resolution: {integrity: sha512-+08ghSARvO2W0VxYh5HO4AnrF7FZ0MC3Shig76HO6shsytMCrgXAvYwbfBufP2bd0HR9eI1a5J7xKazjXX7ogA==}resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}@@ -2838,8 +2838,8 @@ packages:resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}engines: {node: '>= 0.10'}- resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}+ resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==}engines: {node: '>=12.22.0'}@@ -6880,10 +6880,10 @@ snapshots:base64-js: 1.5.1ieee754: 1.2.1dependencies:cron-parser: 4.9.0- ioredis: 5.6.1+ ioredis: 5.4.1msgpackr: 1.11.2node-abort-controller: 3.1.1semver: 7.6.2@@ -7711,7 +7711,7 @@ snapshots:dependencies:agent-base: 7.1.3- debug: 4.4.1([email protected])+ debug: 4.3.5transitivePeerDependencies:- supports-color@@ -7767,11 +7767,11 @@ snapshots:dependencies:'@ioredis/commands': 1.2.0cluster-key-slot: 1.1.2- debug: 4.4.1([email protected])+ debug: 4.3.5denque: 2.1.0lodash.defaults: 4.2.0lodash.isarguments: 3.1.0diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 1769edc95..21c19445c 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -86,7 +86,6 @@ export async function scrapeController(jobPriority,isDirectToBullMQ,);- logger.info("Added scrape job now")const totalWait =(req.body.waitFor ?? 0) +diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 768b63a96..1de240c5d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -143,9 +143,7 @@ export async function fireEngineCheckStatus(},},async (span) => {- const x = Date.now();- meta.logger.debug("Checking status (" + x + ")");- const res = await robustFetch({+ return await robustFetch({url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,method: "GET",logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),@@ -160,8 +158,6 @@ export async function fireEngineCheckStatus(mock,abort,});- meta.logger.debug("Status came back (" + x + ")");- return res;},);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 4cd7ba80a..6e87023f6 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -392,18 +392,15 @@ export function waitForJob(): Promise<Document> {return new Promise(async (resolve, reject) => {const start = Date.now();- const queue = getScrapeQueue();while (true) {- logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });if (Date.now() >= start + timeout) {reject(new Error("Job wait "));break;} else {- const state = await queue.getJobState(jobId);- logger.debug("Job in state", { state, scrapeId: jobId, jobId });+ const state = await getScrapeQueue().getJobState(jobId);if (state === "completed") {let doc: Document;- const job = (await queue.getJob(jobId))!;+ const job = (await getScrapeQueue().getJob(jobId))!;logger.debug("Got job");doc = job.returnvalue;@@ -423,7 +420,7 @@ export function waitForJob(resolve(doc);break;} else if (state === "failed") {- const job = await queue.getJob(jobId);+ const job = await getScrapeQueue().getJob(jobId);if (job && job.failedReason !== "Concurrency limit hit") {reject(job.failedReason);break;@@ -431,7 +428,7 @@ export function waitForJob(}}- await new Promise((resolve) => setTimeout(resolve, 750));+ await new Promise((resolve) => setTimeout(resolve, 250));}return;