Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(precrawl): useIndex for new index func

delong3aa32718e8634
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 04e879be3..e56fcd7d5 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -920,6 +920,10 @@ export async function queryDomainsForPrecrawl(
maxDomains = 50,
logger: Logger = _logger,
): Promise<DomainPriority[]> {
+ if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {
+ return [];
+ }
+
let results: DomainPriority[] = [];
let iteration = 0;
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index f19f83f53..e662e999b 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -135,7 +135,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const DOMAIN_ONLY_RUN = false;
const MAX_PRE_CRAWL_BUDGET = 10000; // maximum number of pages to precrawl this job
- const MIN_PAGE_BUDGET = 25; // minimum budget per page
const MAX_PRE_CRAWL_DOMAINS = 100; // maximum number of domains to precrawl
const MIN_DOMAIN_PRIORITY = 2.0; // minimum priority score to consider a domain
@@ -156,7 +155,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {
"precrawl.team_id": teamId,
"precrawl.dry_run": DRY_RUN,
"precrawl.config.budget": MAX_PRE_CRAWL_BUDGET,
- "precrawl.config.min_page_budget": MIN_PAGE_BUDGET,
"precrawl.config.max_domains": MAX_PRE_CRAWL_DOMAINS,
"precrawl.config.min_priority": MIN_DOMAIN_PRIORITY,
"precrawl.config.min_events": MIN_DOMAIN_EVENTS,
@@ -382,9 +380,8 @@ const processPrecrawlJob = async (token: string, job: Job) => {
const domainBudget = (domain.priority / totalPriority) * totalBudget;
for (const page of filteredPages) {
- const pageBudget = Math.max(
- Math.round((page.event_count / totalEvents) * domainBudget),
- MIN_PAGE_BUDGET,
+ const pageBudget = Math.round(
+ (page.event_count / totalEvents) * domainBudget,
);
const rootUrl = `https://${page.domain}/`;