Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

ci: fix E2E tests (#3855)

Martin Adámek3e3dc6389622
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 7662929a2..d0e460cde 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -87,6 +87,8 @@ jobs:
run: yarn test:e2e
env:
STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
+ # Provided explicitly: the CLI may store the token in the OS keyring, where the tests can't read it.
+ APIFY_TOKEN: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
GITHUB_TOKEN: ${{ github.token }}
diff --git a/test/e2e/adaptive-playwright-robots-file/actor/main.js b/test/e2e/adaptive-playwright-robots-file/actor/main.js
index 96a87f868..d3355bc1f 100644
--- a/test/e2e/adaptive-playwright-robots-file/actor/main.js
+++ b/test/e2e/adaptive-playwright-robots-file/actor/main.js
@@ -9,6 +9,8 @@ await Actor.init({
});
const crawler = new AdaptivePlaywrightCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 10,
respectRobotsTxtFile: true,
onSkippedRequest: (args) => crawler.log.warningOnce(`Request ${args.url} was skipped, reason: ${args.reason}`),
diff --git a/test/e2e/cheerio-robots-file/actor/main.js b/test/e2e/cheerio-robots-file/actor/main.js
index db191d417..820f4be4b 100644
--- a/test/e2e/cheerio-robots-file/actor/main.js
+++ b/test/e2e/cheerio-robots-file/actor/main.js
@@ -9,6 +9,8 @@ await Actor.init({
});
const crawler = new CheerioCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 10,
respectRobotsTxtFile: true,
});
diff --git a/test/e2e/playwright-introduction-guide/actor/main.js b/test/e2e/playwright-introduction-guide/actor/main.js
index 5d232eed6..f6cbd281f 100644
--- a/test/e2e/playwright-introduction-guide/actor/main.js
+++ b/test/e2e/playwright-introduction-guide/actor/main.js
@@ -89,6 +89,8 @@ router.addDefaultHandler(async ({ request, page, enqueueLinks, log }) => {
});
const crawler = new PlaywrightCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 15, // so the test runs faster
// Instead of the long requestHandler with
// if clauses we provide a router instance.
diff --git a/test/e2e/playwright-robots-file/actor/main.js b/test/e2e/playwright-robots-file/actor/main.js
index f38eb3e49..c50ba6939 100644
--- a/test/e2e/playwright-robots-file/actor/main.js
+++ b/test/e2e/playwright-robots-file/actor/main.js
@@ -9,6 +9,8 @@ await Actor.init({
});
const crawler = new PlaywrightCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 10,
respectRobotsTxtFile: true,
});
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
index 635d6f911..c78ef8241 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
@@ -11,6 +11,8 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 10,
preNavigationHooks: [
async ({ page }, goToOptions) => {
diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.js
index 85e4a355e..c9e1b0da7 100644
--- a/test/e2e/puppeteer-store-pagination/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination/actor/main.js
@@ -9,6 +9,8 @@ await Actor.init({
});
const crawler = new PuppeteerCrawler({
+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.
+ proxyConfiguration: await Actor.createProxyConfiguration(),
maxRequestsPerCrawl: 10,
preNavigationHooks: [
async ({ page }, goToOptions) => {
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index be30e5480..c47615d26 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -339,6 +339,15 @@ export async function getApifyToken() {
}
const { token } = await fs.readJSON(authPath);
+
+ // Newer CLI versions keep the token in the OS keyring, leaving auth.json without it.
+ if (!token) {
+ throw new Error(
+ 'Your Apify token is not stored in auth.json (the CLI likely saved it in the OS keyring). ' +
+ 'Set the "APIFY_TOKEN" environment variable, or re-run "apify login" with "APIFY_DISABLE_KEYRING=1".',
+ );
+ }
+
return token;
}