Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
debug(queue-worker): log currently running jobs (non-zdr)
Gergő Móriczc04571d56277
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 0f1eb0535..de96f194b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1684,6 +1684,14 @@ app.listen(workerPort, () => {await new Promise((resolve) => setTimeout(resolve, 500));}+ setInterval(async () => {+ _logger.debug("Currently running jobs", {+ jobs: (await Promise.all([...runningJobs].map(async (jobId) => {+ return await getScrapeQueue().getJob(jobId);+ }))).filter(x => x && !x.data?.zeroDataRetention),+ });+ }, 1000);+await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");process.exit(0);diff --git a/apps/api/src/services/system-monitor.ts b/apps/api/src/services/system-monitor.tsindex c0e23f89f..a2de61632 100644--- a/apps/api/src/services/system-monitor.ts+++ b/apps/api/src/services/system-monitor.ts@@ -228,40 +228,3 @@ class SystemMonitor {}export default SystemMonitor.getInstance();--import { exec } from 'child_process';-import { promisify } from 'util';--const execAsync = promisify(exec);--const testInterval = setInterval(async () => {- try {- // Get top 10 processes by memory usage- const { stdout } = await execAsync('ps aux --no-headers --sort=-rss | head -n 10');-- // Parse the output into an array of objects- const processes = stdout.trim().split('\n').map(line => {- // Remove leading/trailing whitespace and collapse multiple spaces- const parts = line.trim().replace(/\s+/g, ' ').split(' ');-- // ps aux format: USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND- const [user, pid, cpu, mem, vsz, rss, tty, stat, start, time, ...cmdParts] = parts;-- // Convert RSS from KB to GB- const memoryGb = parseFloat(rss) / 1024 / 1024;-- return {- pid: parseInt(pid, 10),- user,- memoryGb: memoryGb.toFixed(2),- command: cmdParts.join(' ') || '(unknown)'- };- });-- logger.debug('Top 10 processes by memory usage', { processes });- } catch (error) {- logger.error('Error getting process memory usage', error);- }-}, 1000); // Run every 1 second--testInterval.unref();\ No newline at end of file