Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "revert all haphazard changes made while bughunting"

Gergő Móricz50e88cfcb7a1
diff --git a/apps/api/package.json b/apps/api/package.json
index 337409d55..28b1bc5c9 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -82,7 +82,7 @@
"axios-retry": "^4.5.0",
"body-parser": "^1.20.3",
"bottleneck": "^2.19.5",
- "bullmq": "^5.36.0",
+ "bullmq": "^5.56.7",
"cacheable-lookup": "^6.1.0",
"cheerio": "^1.0.0-rc.12",
"cohere": "^1.1.1",
@@ -100,7 +100,7 @@
"glob": "^10.4.2",
"gpt3-tokenizer": "^1.1.5",
"http-cookie-agent": "^7.0.1",
- "ioredis": "^5.4.1",
+ "ioredis": "^5.6.1",
"ip-address": "^10.0.1",
"joplin-turndown-plugin-gfm": "^1.0.12",
"jsdom": "^26.0.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index db4b618f7..82854b6c8 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -102,8 +102,8 @@ importers:
specifier: ^2.19.5
version: 2.19.5
bullmq:
- specifier: ^5.36.0
- version: 5.36.0
+ specifier: ^5.56.7
+ version: 5.56.7
cacheable-lookup:
specifier: ^6.1.0
version: 6.1.0
@@ -156,8 +156,8 @@ importers:
specifier: ^7.0.1
ioredis:
- specifier: ^5.4.1
- version: 5.4.1
+ specifier: ^5.6.1
+ version: 5.6.1
ip-address:
specifier: ^10.0.1
version: 10.0.1
@@ -2016,8 +2016,8 @@ packages:
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
- resolution: {integrity: sha512-+08ghSARvO2W0VxYh5HO4AnrF7FZ0MC3Shig76HO6shsytMCrgXAvYwbfBufP2bd0HR9eI1a5J7xKazjXX7ogA==}
+ resolution: {integrity: sha512-Aa4Y7rmkuZOuyHvitGd2rSETgQ0SMawPD0XVsU9gAUg9Q4bwNpSD9ZDVcWGhfAQAUZFJH82JUmLsm0B59Prmtg==}
resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
@@ -2838,8 +2838,8 @@ packages:
resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}
engines: {node: '>= 0.10'}
- resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==}
+ resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}
engines: {node: '>=12.22.0'}
@@ -6880,10 +6880,10 @@ snapshots:
base64-js: 1.5.1
ieee754: 1.2.1
dependencies:
cron-parser: 4.9.0
- ioredis: 5.4.1
+ ioredis: 5.6.1
msgpackr: 1.11.2
node-abort-controller: 3.1.1
semver: 7.6.2
@@ -7711,7 +7711,7 @@ snapshots:
dependencies:
agent-base: 7.1.3
- debug: 4.3.5
+ debug: 4.4.1([email protected])
transitivePeerDependencies:
- supports-color
@@ -7767,11 +7767,11 @@ snapshots:
dependencies:
'@ioredis/commands': 1.2.0
cluster-key-slot: 1.1.2
- debug: 4.3.5
+ debug: 4.4.1([email protected])
denque: 2.1.0
lodash.defaults: 4.2.0
lodash.isarguments: 3.1.0
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 21c19445c..1769edc95 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -86,6 +86,7 @@ export async function scrapeController(
jobPriority,
isDirectToBullMQ,
);
+ logger.info("Added scrape job now")
const totalWait =
(req.body.waitFor ?? 0) +
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 1de240c5d..768b63a96 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -143,7 +143,9 @@ export async function fireEngineCheckStatus(
},
},
async (span) => {
- return await robustFetch({
+ const x = Date.now();
+ meta.logger.debug("Checking status (" + x + ")");
+ const res = await robustFetch({
url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,
method: "GET",
logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),
@@ -158,6 +160,8 @@ export async function fireEngineCheckStatus(
mock,
abort,
});
+ meta.logger.debug("Status came back (" + x + ")");
+ return res;
},
);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 6e87023f6..4cd7ba80a 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -392,15 +392,18 @@ export function waitForJob(
): Promise<Document> {
return new Promise(async (resolve, reject) => {
const start = Date.now();
+ const queue = getScrapeQueue();
while (true) {
+ logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });
if (Date.now() >= start + timeout) {
reject(new Error("Job wait "));
break;
} else {
- const state = await getScrapeQueue().getJobState(jobId);
+ const state = await queue.getJobState(jobId);
+ logger.debug("Job in state", { state, scrapeId: jobId, jobId });
if (state === "completed") {
let doc: Document;
- const job = (await getScrapeQueue().getJob(jobId))!;
+ const job = (await queue.getJob(jobId))!;
logger.debug("Got job");
doc = job.returnvalue;
@@ -420,7 +423,7 @@ export function waitForJob(
resolve(doc);
break;
} else if (state === "failed") {
- const job = await getScrapeQueue().getJob(jobId);
+ const job = await queue.getJob(jobId);
if (job && job.failedReason !== "Concurrency limit hit") {
reject(job.failedReason);
break;
@@ -428,7 +431,7 @@ export function waitForJob(
}
}
- await new Promise((resolve) => setTimeout(resolve, 250));
+ await new Promise((resolve) => setTimeout(resolve, 750));
}
return;