Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import fs from 'node:fs';
import path from 'node:path';
import { BaseHttpClient } from '@crawlee/http-client';
import { URL_WITH_COMMAS_REGEX } from '@crawlee/utils/internal';
import { downloadListOfUrls, extractUrls } from '@crawlee/utils';
const baseDataPath = path.join(import.meta.dirname, '..', 'shared', 'data');
describe('downloadListOfUrls()', () => {
test('downloads a list of URLs', async () => {
const text = fs.readFileSync(path.join(baseDataPath, 'simple_url_list.txt'), 'utf8');
const arr = text
.trim()
.split(/[\r\n]+/g)
.map((u) => u.trim());
const mockClient = Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, {
async sendRequest() {
return new Response(text);
},
});
await expect(
downloadListOfUrls({
url: 'http://www.nowhere12345.com',
httpClient: mockClient,
}),
).resolves.toEqual(arr);
});
});
describe('extractUrls()', () => {
const SIMPLE_URL_LIST = 'simple_url_list.txt';
const UNICODE_URL_LIST = 'unicode_url_list.txt';
const COMMA_URL_LIST = 'unicode+comma_url_list.txt';
const TRICKY_URL_LIST = 'tricky_url_list.txt';
const INVALID_URL_LIST = 'invalid_url_list.txt';
const getURLData = (filename: string) => {
const string = fs.readFileSync(path.join(baseDataPath, filename), 'utf8');
const array = string
.trim()
.split(/[\r\n]+/g)
.map((u) => u.trim());
return { string, array };
};
const makeJSON = ({ string, array }: { string: string; array: string[] }) =>
JSON.stringify({
one: [{ http: string }],
two: array.map((url) => ({ num: 123, url })),
});
const makeCSV = (array: string[], delimiter?: string) =>
array.map((url) => ['ABC', 233, url, '.'].join(delimiter || ',')).join('\n');
const makeText = (array: string[]) => {
const text = fs.readFileSync(path.join(baseDataPath, 'lipsum.txt'), 'utf8').split('');
const ID = 'ů';
const maxIndex = text.length - 1;
array.forEach((__, index) => {
const indexInText = (index * 17) % maxIndex;
if (text[indexInText] === ID) {
text[indexInText + 1] = ID;
} else {
text[indexInText] = ID;
}
});
return array.reduce((string, url) => string.replace(ID, ` ${url} `), text.join(''));
};
test('extracts simple URLs', () => {
const { string, array } = getURLData(SIMPLE_URL_LIST);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs', () => {
const { string, array } = getURLData(UNICODE_URL_LIST);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs with commas', () => {
const { string, array } = getURLData(COMMA_URL_LIST);
const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).toEqual(array);
});
test('extracts tricky URLs', () => {
const { string, array } = getURLData(TRICKY_URL_LIST);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
// https://github.com/apify/crawlee/issues/2755
test('extracts URLs with apostrophes in the path', () => {
const url = "https://www.zillow.com/homedetails/141-O'Canoe-Pl-Hertford-NC-27944/74398007_zpid/";
const extracted = extractUrls({ string: url });
expect(extracted).toEqual([url]);
});
test('extracts URLs with asterisks in the path', () => {
const url = 'https://example.com/path*star/end';
const extracted = extractUrls({ string: url });
expect(extracted).toEqual([url]);
});
test('extracts URLs with internationalized top-level domains', () => {
const urls = [
'https://пример.рф/путь',
'https://xn--e1afmkfd.xn--p1ai/path',
'https://例子.中国/',
'https://उदाहरण.भारत/page',
'https://test.みんな/',
];
expect(extractUrls({ string: urls.join(' ') })).toEqual(urls);
expect(extractUrls({ string: urls.join(' '), urlRegExp: URL_WITH_COMMAS_REGEX })).toEqual(urls);
});
test('does not extend an ASCII top-level domain into adjacent non-Latin text', () => {
const string = '詳細はhttps://example.comをご覧ください 请访问https://www.google.com获取更多信息';
const expected = ['https://example.com', 'https://www.google.com'];
expect(extractUrls({ string })).toEqual(expected);
expect(extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX })).toEqual(expected);
});
test('does not extract invalid URLs', () => {
const { string } = getURLData(INVALID_URL_LIST);
const extracted = extractUrls({ string });
expect(extracted).toEqual(['http://www.foo.bar']);
});
test('extracts simple URLs from JSON', () => {
const d = getURLData(SIMPLE_URL_LIST);
const string = makeJSON(d);
const extracted = extractUrls({ string });
expect(extracted).toEqual(d.array.concat(d.array));
});
test('extracts unicode URLs from JSON', () => {
const d = getURLData(UNICODE_URL_LIST);
const string = makeJSON(d);
const extracted = extractUrls({ string });
expect(extracted).toEqual(d.array.concat(d.array));
});
test('extracts unicode URLs with commas from JSON', () => {
const d = getURLData(COMMA_URL_LIST);
const string = makeJSON(d);
const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).toEqual(d.array.concat(d.array));
});
test('extracts tricky URLs from JSON', () => {
const d = getURLData(TRICKY_URL_LIST);
const string = makeJSON(d);
const extracted = extractUrls({ string });
expect(extracted).toEqual(d.array.concat(d.array));
});
test('does not extract invalid URLs from JSON', () => {
const d = getURLData(INVALID_URL_LIST);
const string = makeJSON(d);
const extracted = extractUrls({ string });
expect(extracted).toEqual(['http://www.foo.bar', 'http://www.foo.bar']);
});
test('extracts simple URLs from CSV', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeCSV(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs from CSV', () => {
const { array } = getURLData(UNICODE_URL_LIST);
const string = makeCSV(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs with commas from semicolon CSV', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeCSV(array, ';');
const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).toEqual(array);
});
test('extracts tricky URLs from CSV', () => {
const { array } = getURLData(TRICKY_URL_LIST);
const string = makeCSV(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('does not extract invalid URLs from CSV', () => {
const { array } = getURLData(INVALID_URL_LIST);
const string = makeCSV(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(['http://www.foo.bar']);
});
test('extracts simple URLs from Text', () => {
const { array } = getURLData(SIMPLE_URL_LIST);
const string = makeText(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs from Text', () => {
const { array } = getURLData(UNICODE_URL_LIST);
const string = makeText(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('extracts unicode URLs with commas from Text', () => {
const { array } = getURLData(COMMA_URL_LIST);
const string = makeText(array);
const extracted = extractUrls({ string, urlRegExp: URL_WITH_COMMAS_REGEX });
expect(extracted).toEqual(array);
});
test('extracts tricky URLs from Text', () => {
const { array } = getURLData(TRICKY_URL_LIST);
const string = makeText(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(array);
});
test('does not extract invalid URLs from Text', () => {
const { array } = getURLData(INVALID_URL_LIST);
const string = makeText(array);
const extracted = extractUrls({ string });
expect(extracted).toEqual(['http://www.foo.bar']);
});
});