Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: crawl log parser (poc)
Gergő Móricz877f072e3c51
diff --git a/apps/api/logview.js b/apps/api/logview.jsnew file mode 100644index 000000000..17032b2e4--- /dev/null+++ b/apps/api/logview.js@@ -0,0 +1,23 @@+const fs = require("fs");++const logs = fs.readFileSync("log-20780c8a-52f5-4af7-ac48-62997d11ec9b.log", "utf8")+ .split("\n").filter(x => x.trim().length > 0).map(x => JSON.parse(x));++const crawlIds = [...new Set(logs.map(x => x.crawlId).filter(x => x))];++const urlFilter = x => new URL(x).pathname.slice(1) || "root"++for (const crawlId of crawlIds) {+ const crawlLogs = logs.filter(x => x.crawlId === crawlId);++ const jobAdds = crawlLogs.filter(x => x.jobPriority !== undefined && x.message.startsWith("Added job for URL "));+ const jobStarts = crawlLogs.filter(x => x.message.startsWith("🐂 Worker taking job"));++ fs.writeFileSync(crawlId + ".md",+ "```mermaid\nflowchart LR\n "+ + jobStarts.map(x => `${x.jobId}[${urlFilter(x.url)}]`).join("\n ") + "\n "+ + jobAdds.map(x => `${x.jobId}[${urlFilter(jobStarts.find(y => y.jobId === x.jobId).url)}] --> ${x.newJobId}[${urlFilter(x.url)}]`).join("\n ")+ + "\n```\n\nURLs scraped: (" + jobStarts.length + ")\n"+ + jobStarts.map(x => "- " + x.url).join("\n")+ );+}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 785783956..9d25848b4 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -346,7 +346,7 @@ workerFun(getScrapeQueue(), processJobInternal);async function processJob(job: Job & { id: string }, token: string) {const logger = _logger.child({ module: "queue-worker", method: "processJob", jobId: job.id, scrapeId: job.id, crawlId: job.data?.crawl_id ?? undefined });- logger.info(`🐂 Worker taking job ${job.id}`);+ logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });// Check if the job URL is researchhub and block it immediately// TODO: remove this once solve the root issue@@ -505,7 +505,7 @@ async function processJob(job: Job & { id: string }, token: string) {);await addCrawlJob(job.data.crawl_id, jobId);- logger.debug("Added job for URL " + JSON.stringify(link), { jobPriority, url: link });+ logger.debug("Added job for URL " + JSON.stringify(link), { jobPriority, url: link, newJobId: jobId });} else {logger.debug("Could not lock URL " + JSON.stringify(link), { url: link });}