Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(precrawl): priority nan + rpc null
delong3d8beb94b122e
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex e5ac69d26..c7b840bbc 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -93,7 +93,11 @@ const logger = {console.log(`${colors.yellow}!${colors.reset} ${message}`);},error(message: string, error?: any) {- console.error(`${colors.red}β${colors.reset} ${message}`, error);+ if (error) {+ console.error(`${colors.red}β${colors.reset} ${message}`, error);+ } else {+ console.error(`${colors.red}β${colors.reset} ${message}`);+ }},processStart(name: string, command: string) {const color = getProcessColor(name);diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 7e81e8843..04e879be3 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -943,13 +943,14 @@ export async function queryDomainsForPrecrawl(return results.slice(0, maxDomains);}- results = results.concat(data || []);+ const batchData = data ?? [];+ results = results.concat(batchData);if (results.length >= maxDomains) {return results.slice(0, maxDomains);}- if (data.length < 1000) {+ if (batchData.length < 1000) {return results.slice(0, maxDomains);}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 4015967db..ad947b7bd 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -220,7 +220,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {const domainQueries = domains.map(d => {const normalizedPriority =- (d.priority - minPriority) / (maxPriority - minPriority);+ maxPriority === minPriority+ ? 0+ : (d.priority - minPriority) / (maxPriority - minPriority);const urlsToFetch = Math.round(MIN_URLS_PER_DOMAIN +@@ -542,7 +544,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {let isShuttingDown = false;process.on("SIGINT", () => {- logger.info("Received SIGTERM. Shutting down gracefully...");+ logger.info("Received SIGINT. Shutting down gracefully...");isShuttingDown = true;});