Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(precrawl): priority nan + rpc null

delong3d8beb94b122e
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index e5ac69d26..c7b840bbc 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -93,7 +93,11 @@ const logger = {
console.log(`${colors.yellow}!${colors.reset} ${message}`);
},
error(message: string, error?: any) {
- console.error(`${colors.red}βœ—${colors.reset} ${message}`, error);
+ if (error) {
+ console.error(`${colors.red}βœ—${colors.reset} ${message}`, error);
+ } else {
+ console.error(`${colors.red}βœ—${colors.reset} ${message}`);
+ }
},
processStart(name: string, command: string) {
const color = getProcessColor(name);
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 7e81e8843..04e879be3 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -943,13 +943,14 @@ export async function queryDomainsForPrecrawl(
return results.slice(0, maxDomains);
}
- results = results.concat(data || []);
+ const batchData = data ?? [];
+ results = results.concat(batchData);
if (results.length >= maxDomains) {
return results.slice(0, maxDomains);
}
- if (data.length < 1000) {
+ if (batchData.length < 1000) {
return results.slice(0, maxDomains);
}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 4015967db..ad947b7bd 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -220,7 +220,9 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const domainQueries = domains.map(d => {
const normalizedPriority =
- (d.priority - minPriority) / (maxPriority - minPriority);
+ maxPriority === minPriority
+ ? 0
+ : (d.priority - minPriority) / (maxPriority - minPriority);
const urlsToFetch = Math.round(
MIN_URLS_PER_DOMAIN +
@@ -542,7 +544,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
let isShuttingDown = false;
process.on("SIGINT", () => {
- logger.info("Received SIGTERM. Shutting down gracefully...");
+ logger.info("Received SIGINT. Shutting down gracefully...");
isShuttingDown = true;
});