Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "revert all haphazard changes made while bughunting"
Gergő Móricz50e88cfcb7a1
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 337409d55..28b1bc5c9 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -82,7 +82,7 @@"axios-retry": "^4.5.0","body-parser": "^1.20.3","bottleneck": "^2.19.5",- "bullmq": "^5.36.0",+ "bullmq": "^5.56.7","cacheable-lookup": "^6.1.0","cheerio": "^1.0.0-rc.12","cohere": "^1.1.1",@@ -100,7 +100,7 @@"glob": "^10.4.2","gpt3-tokenizer": "^1.1.5","http-cookie-agent": "^7.0.1",- "ioredis": "^5.4.1",+ "ioredis": "^5.6.1","ip-address": "^10.0.1","joplin-turndown-plugin-gfm": "^1.0.12","jsdom": "^26.0.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex db4b618f7..82854b6c8 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -102,8 +102,8 @@ importers:specifier: ^2.19.5version: 2.19.5bullmq:- specifier: ^5.36.0- version: 5.36.0+ specifier: ^5.56.7+ version: 5.56.7cacheable-lookup:specifier: ^6.1.0version: 6.1.0@@ -156,8 +156,8 @@ importers:specifier: ^7.0.1version: 7.0.1([email protected])([email protected])ioredis:- specifier: ^5.4.1- version: 5.4.1+ specifier: ^5.6.1+ version: 5.6.1ip-address:specifier: ^10.0.1version: 10.0.1@@ -2016,8 +2016,8 @@ packages:resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}- resolution: {integrity: sha512-+08ghSARvO2W0VxYh5HO4AnrF7FZ0MC3Shig76HO6shsytMCrgXAvYwbfBufP2bd0HR9eI1a5J7xKazjXX7ogA==}+ resolution: {integrity: sha512-Aa4Y7rmkuZOuyHvitGd2rSETgQ0SMawPD0XVsU9gAUg9Q4bwNpSD9ZDVcWGhfAQAUZFJH82JUmLsm0B59Prmtg==}resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}@@ -2838,8 +2838,8 @@ packages:resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}engines: {node: '>= 0.10'}- resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==}+ resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}engines: {node: '>=12.22.0'}@@ -6880,10 +6880,10 @@ snapshots:base64-js: 1.5.1ieee754: 1.2.1dependencies:cron-parser: 4.9.0- ioredis: 5.4.1+ ioredis: 5.6.1msgpackr: 1.11.2node-abort-controller: 3.1.1semver: 7.6.2@@ -7711,7 +7711,7 @@ snapshots:dependencies:agent-base: 7.1.3- debug: 4.3.5+ debug: 4.4.1([email protected])transitivePeerDependencies:- supports-color@@ -7767,11 +7767,11 @@ snapshots:dependencies:'@ioredis/commands': 1.2.0cluster-key-slot: 1.1.2- debug: 4.3.5+ debug: 4.4.1([email protected])denque: 2.1.0lodash.defaults: 4.2.0lodash.isarguments: 3.1.0diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 21c19445c..1769edc95 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -86,6 +86,7 @@ export async function scrapeController(jobPriority,isDirectToBullMQ,);+ logger.info("Added scrape job now")const totalWait =(req.body.waitFor ?? 0) +diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 1de240c5d..768b63a96 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -143,7 +143,9 @@ export async function fireEngineCheckStatus(},},async (span) => {- return await robustFetch({+ const x = Date.now();+ meta.logger.debug("Checking status (" + x + ")");+ const res = await robustFetch({url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,method: "GET",logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),@@ -158,6 +160,8 @@ export async function fireEngineCheckStatus(mock,abort,});+ meta.logger.debug("Status came back (" + x + ")");+ return res;},);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 6e87023f6..4cd7ba80a 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -392,15 +392,18 @@ export function waitForJob(): Promise<Document> {return new Promise(async (resolve, reject) => {const start = Date.now();+ const queue = getScrapeQueue();while (true) {+ logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });if (Date.now() >= start + timeout) {reject(new Error("Job wait "));break;} else {- const state = await getScrapeQueue().getJobState(jobId);+ const state = await queue.getJobState(jobId);+ logger.debug("Job in state", { state, scrapeId: jobId, jobId });if (state === "completed") {let doc: Document;- const job = (await getScrapeQueue().getJob(jobId))!;+ const job = (await queue.getJob(jobId))!;logger.debug("Got job");doc = job.returnvalue;@@ -420,7 +423,7 @@ export function waitForJob(resolve(doc);break;} else if (state === "failed") {- const job = await getScrapeQueue().getJob(jobId);+ const job = await queue.getJob(jobId);if (job && job.failedReason !== "Concurrency limit hit") {reject(job.failedReason);break;@@ -428,7 +431,7 @@ export function waitForJob(}}- await new Promise((resolve) => setTimeout(resolve, 250));+ await new Promise((resolve) => setTimeout(resolve, 750));}return;