Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(precrawl): testing config + profile debug
delong3da54401c84df
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex efcbfdb96..9033f5717 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -114,7 +114,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {return err;};-// TODO: cron job for triggering this + updating index+// NOTE: current config is 100 domains with 250 urls per domain with estimated max budget of 10,000const processPrecrawlJob = async (token: string, job: Job) => {const logger = _logger.child({module: "precrawl-worker",@@ -127,7 +127,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {}, jobLockExtendInterval);// set to true to prevent crawl job submissions- const DRY_RUN = true;+ const DRY_RUN = false;// set to true to only run domain precrawl, no individual URLs or crawl jobsconst DOMAIN_ONLY_RUN = false;@@ -135,17 +135,32 @@ const processPrecrawlJob = async (token: string, job: Job) => {const MAX_PRE_CRAWL_BUDGET = 10000; // maximum number of pages to precrawl this jobconst MIN_PAGE_BUDGET = 25; // minimum budget per page- const MAX_PRE_CRAWL_DOMAINS = 2000; // maximum number of domains to precrawl+ const MAX_PRE_CRAWL_DOMAINS = 100; // maximum number of domains to precrawlconst MIN_DOMAIN_PRIORITY = 2.0; // minimum priority score to consider a domainconst MIN_DOMAIN_EVENTS = 1000; // minimum number of events to consider a domain- const DOMAIN_URL_BATCH_SIZE = 25; // number of domain hashes to query in parallel+ // number of domain hashes to query in parallel - keep relatively low for now (25 is good)+ const DOMAIN_URL_BATCH_SIZE = 25;const MIN_URLS_PER_DOMAIN = 10;- const MAX_URLS_PER_DOMAIN = 500;+ const MAX_URLS_PER_DOMAIN = 250;const teamId = process.env.PRECRAWL_TEAM_ID;+ logger.info("Received index pre-crawl trigger job", {+ teamId,+ config: {+ MAX_PRE_CRAWL_BUDGET,+ MIN_PAGE_BUDGET,+ MAX_PRE_CRAWL_DOMAINS,+ MIN_DOMAIN_PRIORITY,+ MIN_DOMAIN_EVENTS,+ DOMAIN_URL_BATCH_SIZE,+ MIN_URLS_PER_DOMAIN,+ MAX_URLS_PER_DOMAIN,+ },+ });+try {await withSpan("precrawl.job", async span => {setSpanAttributes(span, {@@ -154,8 +169,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {"precrawl.dry_run": DRY_RUN,});- await new Promise(resolve => setTimeout(resolve, 20000));-const dateFuture = new Date();dateFuture.setHours(dateFuture.getHours() + 1);@@ -233,7 +246,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {`Total budget: ${totalBudget} for ${domains.length} domains`,);logger.debug(`------------------------------`);- console.log(+ logger.debug(`Precrawl domains: (${domainQueries.length}) ${JSON.stringify(domainQueries.slice(0, 5), null, 2)} ...`,);await job.moveToCompleted({ success: true }, token, false);@@ -270,19 +283,22 @@ const processPrecrawlJob = async (token: string, job: Job) => {.overrideTypes<DomainUrlResult[]>();});+ const startTimeNs = process.hrtime.bigint();const batchResults = (await Promise.allSettled(batchFutures,)) as PromiseSettledResult<{data: DomainUrlResult[] | null;error: any;}>[];+ const endTimeNs = process.hrtime.bigint();urlResults.push(...batchResults);if (i < batches.length - 1) {+ const durationMs = Number(endTimeNs - startTimeNs) / 1e6;const backoff = Math.min(1000 + i * 100, 3000);logger.debug(- `Completed batch ${i + 1}/${batches.length} of URL fetches (failed: ${batchResults.filter(r => r.status === "rejected" || (r.status === "fulfilled" && r.value.error)).length})... Waiting for ${backoff}ms before next batch...`,+ `Completed batch ${i + 1}/${batches.length} of URL fetches (failed: ${batchResults.filter(r => r.status === "rejected" || (r.status === "fulfilled" && r.value.error)).length}) in ${durationMs}ms. Waiting for ${backoff}ms before next batch...`,);await new Promise(resolve => setTimeout(resolve, backoff));}