Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

3.0 KB
Raw Blame
import { describe, expect, it } from 'vitest';

import { filterUrl, matchesEnqueueStrategy } from '../../packages/utils/src/internals/url.js';

describe('matchesEnqueueStrategy', () => {
    const match = (strategy: Parameters<typeof matchesEnqueueStrategy>[0], target: string, origin: string) =>
        matchesEnqueueStrategy(strategy, new URL(target), new URL(origin));

    it('all matches regardless of host, domain, or scheme', () => {
        expect(match('all', 'https://other.test/page', 'http://example.com/sitemap.xml')).toBe(true);
    });

    it('same-hostname matches only the exact host, regardless of port', () => {
        expect(match('same-hostname', 'http://example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
        expect(match('same-hostname', 'http://example.com:8080/a', 'http://example.com/sitemap.xml')).toBe(true);
        expect(match('same-hostname', 'http://www.example.com/a', 'http://example.com/sitemap.xml')).toBe(false);
    });

    it('same-domain matches across subdomains but not across registrable domains', () => {
        expect(match('same-domain', 'http://www.example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
        expect(match('same-domain', 'http://example.org/a', 'http://example.com/sitemap.xml')).toBe(false);
    });

    it('same-domain on a host without a registrable domain falls back to same-hostname', () => {
        expect(match('same-domain', 'https://10.0.0.5/a', 'http://10.0.0.5/sitemap.xml')).toBe(true);
        expect(match('same-domain', 'http://localhost:3001/a', 'https://localhost:3000/sitemap.xml')).toBe(true);
        expect(match('same-domain', 'http://10.0.0.6/a', 'http://10.0.0.5/sitemap.xml')).toBe(false);
    });

    it('same-origin requires matching scheme, host, and port', () => {
        expect(match('same-origin', 'http://example.com/a', 'http://example.com/sitemap.xml')).toBe(true);
        expect(match('same-origin', 'https://example.com/a', 'http://example.com/sitemap.xml')).toBe(false);
        expect(match('same-origin', 'http://example.com:8080/a', 'http://example.com/sitemap.xml')).toBe(false);
    });

    it('treats a trailing-dot FQDN host as equal to the bare host across strategies', () => {
        for (const strategy of ['same-hostname', 'same-domain', 'same-origin'] as const) {
            expect(match(strategy, 'http://example.com./page', 'http://example.com/sitemap.xml')).toBe(true);
        }
    });
});

describe('filterUrl', () => {
    it('drops non-http(s) schemes regardless of strategy', () => {
        for (const target of ['mailto:[email protected]', 'ftp://example.com/f.txt']) {
            expect(filterUrl(target, 'http://example.com/sitemap.xml', 'all').allowed).toBe(false);
        }
    });

    it('drops gracefully (no throw) when the origin is not parseable', () => {
        expect(() => filterUrl('http://example.com/a', 'not-a-url', 'same-hostname')).not.toThrow();
        expect(filterUrl('http://example.com/a', 'not-a-url', 'same-hostname')).toEqual({
            allowed: false,
            reason: 'invalid origin URL',
        });
    });
});