Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import {
applyRequestTransform,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromRegExps,
constructUrlPatternObjects,
createRequestOptions,
filterRequestOptionsByPatterns,
validateGlobPattern,
} from '@crawlee/basic';
import type { RequestOptions } from '@crawlee/core';
describe('Enqueue links shared functions', () => {
describe('constructGlobObjectsFromGlobs()', () => {
test('should work', () => {
const globs = ['https://example.com/**/*', { glob: '?(http|https)://cool.com/' }];
const globObjects = constructGlobObjectsFromGlobs(globs);
expect(globObjects).toHaveLength(2);
expect(globObjects[0].glob).toEqual('https://example.com/**/*');
expect(globObjects[1].glob).toEqual('?(http|https)://cool.com/');
});
});
describe('constructRegExpObjectsFromRegExps()', () => {
test('should work', () => {
const regexps = [/^https:\/\/example\.com\/(\w|\/)+/, { regexp: /^(http|https):\/\/cool\.com\// }];
const regexpObjects = constructRegExpObjectsFromRegExps(regexps);
expect(regexpObjects).toHaveLength(2);
expect(regexpObjects[0].regexp.test('https://example.com/')).toBe(false);
expect(regexpObjects[1].regexp.test('https://cool.com/')).toBe(true);
});
});
describe('constructUrlPatternObjects()', () => {
test('should handle mixed glob and regexp patterns', () => {
const patterns = [
'https://example.com/**/*',
{ glob: 'https://cool.com/**' },
/^https:\/\/foo\.com/,
{ regexp: /bar\.com/ },
];
const objects = constructUrlPatternObjects(patterns);
expect(objects).toHaveLength(4);
expect(objects[0]).toHaveProperty('glob', 'https://example.com/**/*');
expect(objects[1]).toHaveProperty('glob', 'https://cool.com/**');
expect(objects[2]).toHaveProperty('regexp');
expect(objects[3]).toHaveProperty('regexp');
});
});
describe('caching', () => {
test('should cache items', () => {
const globs0 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
const globs1 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
expect(globs0[0]).toBe(globs1[0]);
// RegExps are cached by instance, so the same one has to be passed twice
const regexp = /^https:\/\/example\.com\/(\w|\/)+/;
const regexps0 = constructRegExpObjectsFromRegExps([regexp]);
const regexps1 = constructRegExpObjectsFromRegExps([regexp]);
expect(regexps0[0]).toBe(regexps1[0]);
});
});
describe('filterRequestOptionsByPatterns() + applyRequestTransform()', () => {
test('should filter by patterns and apply transform', () => {
const sources = [
'http://example.com/foo',
{ url: 'https://example.com/bar', method: 'POST' as const, label: 'POST-REQUEST' },
'https://apify.com',
];
const urlPatternObjects = constructUrlPatternObjects([/^https?:\/\/example\.com\/.*/]);
const transformRequestFunction = (request: RequestOptions) => {
request.userData = { ...request.userData, foo: 'bar' };
return request;
};
const requestOptions = createRequestOptions(sources);
const filtered = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects);
const transformed = applyRequestTransform(filtered, transformRequestFunction);
expect(transformed).toHaveLength(2);
transformed.forEach((r) => {
expect(r.url).toMatch(/^https?:\/\/example\.com\//);
expect(r.userData).toMatchObject({ foo: 'bar' });
});
expect(transformed[0].method).toBeUndefined(); // defaults to GET when Request is constructed
expect(transformed[1].method).toBe('POST');
});
});
describe('validateGlobPattern()', () => {
test('should throw for empty glob patterns', () => {
const globPattern = 'https://example.com/**/*';
expect(() => validateGlobPattern(globPattern)).not.toThrow();
const emptyGlobPattern = '';
expect(() => validateGlobPattern(emptyGlobPattern)).toThrow(
/Cannot parse Glob pattern '': it must be an non-empty string/,
);
});
});
});