Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: add `discoverValidSitemaps` utility (#3392)

Jindřich Bärc6ab237b74af
diff --git a/packages/utils/src/internals/sitemap.ts b/packages/utils/src/internals/sitemap.ts
index 39e360bf8..de57d9bb0 100644
--- a/packages/utils/src/internals/sitemap.ts
+++ b/packages/utils/src/internals/sitemap.ts
@@ -551,9 +551,19 @@ export async function* discoverValidSitemaps(
* Defaults to `20000` ms (20 seconds).
*/
requestTimeoutMillis?: number;
+ /**
+ * HTTP client to be used for network requests.
+ */
+ httpClient?: BaseHttpClient;
} = {},
): AsyncIterable<string> {
- const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000 } = options;
+ const {
+ proxyUrl,
+ timeoutMillis = 60_000,
+ signal: externalSignal,
+ requestTimeoutMillis = 20_000,
+ httpClient = new FetchHttpClient(),
+ } = options;
const controller = new AbortController();
const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
@@ -567,7 +577,6 @@ export async function* discoverValidSitemaps(
}
const signal = controller.signal;
- const { gotScraping } = await import('got-scraping');
const sitemapUrls = new Set<string>();
const addSitemapUrl = (url: string): string | undefined => {
@@ -582,18 +591,16 @@ export async function* discoverValidSitemaps(
return undefined;
};
- const urlExists = async (url: string) => {
- const response = await gotScraping({
- url,
- method: 'HEAD',
- proxyUrl,
- timeout: {
- request: requestTimeoutMillis,
- },
- signal,
- });
-
- return response.statusCode >= 200 && response.statusCode < 400;
+ const urlExists = async (url: string): Promise<boolean> => {
+ if (!httpClient) {
+ return false;
+ }
+ try {
+ const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), { proxyUrl });
+ return response.status >= 200 && response.status < 400;
+ } catch {
+ return false;
+ }
};
const discoverSitemapsForDomainUrls = async function* (hostname: string, domainUrls: string[]) {
@@ -602,7 +609,8 @@ export async function* discoverValidSitemaps(
}
try {
- const robotsFile = await RobotsFile.find(domainUrls[0], proxyUrl, {
+ const robotsFile = await RobotsFile.find(domainUrls[0], {
+ proxyUrl,
timeoutMillis: requestTimeoutMillis,
signal,
});
@@ -661,8 +669,14 @@ export async function* discoverValidSitemaps(
discoverSitemapsForDomainUrls(hostname, domainUrls),
);
+ const discoveredUrls = new Set<string>();
+
try {
for await (const url of mergeAsyncIterables(...iterables)) {
+ if (discoveredUrls.has(url)) {
+ continue;
+ }
+ discoveredUrls.add(url);
yield url;
}
} finally {