Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add test for relative path

Ondra Urbanafd59da5b76e
diff --git a/test/enqueue_links.js b/test/enqueue_links.js
index 1fbcc35b0..3753a02fb 100644
--- a/test/enqueue_links.js
+++ b/test/enqueue_links.js
@@ -25,7 +25,8 @@ const HTML = `
Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
</p>
- <a href="/x/relative">This is a relative link.</a>
+ <a href="/x/absolutepath">This is a relative link.</a>
+ <a href="y/relativepath">This is a relative link.</a>
</body>
</html>
`;
@@ -600,9 +601,9 @@ describe('enqueueLinks()', () => {
enqueued.push(request);
};
- await enqueueLinks({ $, requestQueue, baseUrl: 'http://www.absolute.com/removethis' });
+ await enqueueLinks({ $, requestQueue, baseUrl: 'http://www.absolute.com/removethis/' });
- expect(enqueued).to.have.lengthOf(6);
+ expect(enqueued).to.have.lengthOf(7);
expect(enqueued[0].url).to.be.eql('https://example.com/a/b/first');
expect(enqueued[0].method).to.be.eql('GET');
@@ -624,9 +625,13 @@ describe('enqueueLinks()', () => {
expect(enqueued[4].method).to.be.eql('GET');
expect(enqueued[4].userData).to.be.eql({});
- expect(enqueued[5].url).to.be.eql('http://www.absolute.com/x/relative');
+ expect(enqueued[5].url).to.be.eql('http://www.absolute.com/x/absolutepath');
expect(enqueued[5].method).to.be.eql('GET');
expect(enqueued[5].userData).to.be.eql({});
+
+ expect(enqueued[6].url).to.be.eql('http://www.absolute.com/removethis/y/relativepath');
+ expect(enqueued[6].method).to.be.eql('GET');
+ expect(enqueued[6].userData).to.be.eql({});
});
it('throws on finding a relative link with no baseUrl set', async () => {
@@ -639,7 +644,7 @@ describe('enqueueLinks()', () => {
await enqueueLinks({ $, requestQueue });
throw new Error('wrong error');
} catch (err) {
- expect(err.message).to.include('/x/relative');
+ expect(err.message).to.include('/x/absolutepath');
}
expect(enqueued).to.have.lengthOf(0);
});