Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Use configurable public host for self-hosted response URLs (#3699)
Saivedant Hava2dd97c64927a
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 9c3dc90a1..221214880 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -255,7 +255,7 @@ export async function batchScrapeController(return res.status(200).json({success: true,id,- url: `${protocol}://${req.get("host")}/v1/batch/scrape/${id}`,+ url: `${protocol}://${req.host}/v1/batch/scrape/${id}`,invalidURLs,});}diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex bf4fcd82c..b33a81ba6 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -299,7 +299,7 @@ export async function crawlStatusController(next:(outputBulkA.total ?? 0) > start + iteratedOver ||outputBulkA.status !== "completed"- ? `${req.protocol}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${req.protocol}://${req.host}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 6ae0d21e7..baf3975af 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -202,6 +202,6 @@ export async function crawlController(return res.status(200).json({success: true,id,- url: `${protocol}://${req.get("host")}/v1/crawl/${id}`,+ url: `${protocol}://${req.host}/v1/crawl/${id}`,});}diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 198e209e9..599908dc8 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -285,7 +285,7 @@ export async function batchScrapeController(return res.status(200).json({success: true,id,- url: `${protocol}://${req.get("host")}/v2/batch/scrape/${id}`,+ url: `${protocol}://${req.host}/v2/batch/scrape/${id}`,invalidURLs,});}diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 7c1db8efd..54d053364 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -326,7 +326,7 @@ export async function crawlStatusController(next:(outputBulkA.total ?? 0) > start + iteratedOver ||outputBulkA.status !== "completed"- ? `${req.protocol}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ ? `${req.protocol}://${req.host}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`: undefined,};diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex ae7c9111a..cf9a70820 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -261,7 +261,7 @@ export async function crawlController(return res.status(200).json({success: true,id,- url: `${protocol}://${req.get("host")}/v2/crawl/${id}`,+ url: `${protocol}://${req.host}/v2/crawl/${id}`,...(req.body.prompt && {promptGeneratedOptions: promptGeneratedOptions,finalCrawlerOptions: finalCrawlerOptions,diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex 71d51647c..c69594c96 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -553,7 +553,7 @@ export async function getMonitorCheckController(if (totalPagesForFilter <= nextSkip) return undefined;const url = new URL(`/v2/monitor/${monitorId}/checks/${checkId}`,- `${req.protocol}://${req.get("host")}`,+ `${req.protocol}://${req.host}`,);url.searchParams.set("skip", String(nextSkip));url.searchParams.set("limit", String(query.limit));diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.tsindex 49762600c..9eaac0027 100644--- a/apps/api/src/controllers/v2/parse-upload.ts+++ b/apps/api/src/controllers/v2/parse-upload.ts@@ -172,7 +172,7 @@ function cleanupExpiredLocalUploads(now = Date.now()) {function buildPublicBaseUrl(req: Request): string {const configured = config.PARSE_UPLOAD_PUBLIC_BASE_URL;if (configured) return configured.replace(/\/$/, "");- return `${req.protocol}://${req.get("host")}`;+ return `${req.protocol}://${req.host}`;}function makeObjectPath(