Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(precrawl): useIndex for new index func
delong3aa32718e8634
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 04e879be3..e56fcd7d5 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -920,6 +920,10 @@ export async function queryDomainsForPrecrawl(maxDomains = 50,logger: Logger = _logger,): Promise<DomainPriority[]> {+ if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ return [];+ }+let results: DomainPriority[] = [];let iteration = 0;diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex f19f83f53..e662e999b 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -135,7 +135,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {const DOMAIN_ONLY_RUN = false;const MAX_PRE_CRAWL_BUDGET = 10000; // maximum number of pages to precrawl this job- const MIN_PAGE_BUDGET = 25; // minimum budget per pageconst MAX_PRE_CRAWL_DOMAINS = 100; // maximum number of domains to precrawlconst MIN_DOMAIN_PRIORITY = 2.0; // minimum priority score to consider a domain@@ -156,7 +155,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {"precrawl.team_id": teamId,"precrawl.dry_run": DRY_RUN,"precrawl.config.budget": MAX_PRE_CRAWL_BUDGET,- "precrawl.config.min_page_budget": MIN_PAGE_BUDGET,"precrawl.config.max_domains": MAX_PRE_CRAWL_DOMAINS,"precrawl.config.min_priority": MIN_DOMAIN_PRIORITY,"precrawl.config.min_events": MIN_DOMAIN_EVENTS,@@ -382,9 +380,8 @@ const processPrecrawlJob = async (token: string, job: Job) => {const domainBudget = (domain.priority / totalPriority) * totalBudget;for (const page of filteredPages) {- const pageBudget = Math.max(- Math.round((page.event_count / totalEvents) * domainBudget),- MIN_PAGE_BUDGET,+ const pageBudget = Math.round(+ (page.event_count / totalEvents) * domainBudget,);const rootUrl = `https://${page.domain}/`;