Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
ci: fix E2E tests (#3855)
Martin Adámek3e3dc6389622
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex 7662929a2..d0e460cde 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -87,6 +87,8 @@ jobs:run: yarn test:e2eenv:STORAGE_IMPLEMENTATION: ${{ matrix.storage }}+ # Provided explicitly: the CLI may store the token in the OS keyring, where the tests can't read it.+ APIFY_TOKEN: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}GITHUB_TOKEN: ${{ github.token }}diff --git a/test/e2e/adaptive-playwright-robots-file/actor/main.js b/test/e2e/adaptive-playwright-robots-file/actor/main.jsindex 96a87f868..d3355bc1f 100644--- a/test/e2e/adaptive-playwright-robots-file/actor/main.js+++ b/test/e2e/adaptive-playwright-robots-file/actor/main.js@@ -9,6 +9,8 @@ await Actor.init({});const crawler = new AdaptivePlaywrightCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 10,respectRobotsTxtFile: true,onSkippedRequest: (args) => crawler.log.warningOnce(`Request ${args.url} was skipped, reason: ${args.reason}`),diff --git a/test/e2e/cheerio-robots-file/actor/main.js b/test/e2e/cheerio-robots-file/actor/main.jsindex db191d417..820f4be4b 100644--- a/test/e2e/cheerio-robots-file/actor/main.js+++ b/test/e2e/cheerio-robots-file/actor/main.js@@ -9,6 +9,8 @@ await Actor.init({});const crawler = new CheerioCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 10,respectRobotsTxtFile: true,});diff --git a/test/e2e/playwright-introduction-guide/actor/main.js b/test/e2e/playwright-introduction-guide/actor/main.jsindex 5d232eed6..f6cbd281f 100644--- a/test/e2e/playwright-introduction-guide/actor/main.js+++ b/test/e2e/playwright-introduction-guide/actor/main.js@@ -89,6 +89,8 @@ router.addDefaultHandler(async ({ request, page, enqueueLinks, log }) => {});const crawler = new PlaywrightCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 15, // so the test runs faster// Instead of the long requestHandler with// if clauses we provide a router instance.diff --git a/test/e2e/playwright-robots-file/actor/main.js b/test/e2e/playwright-robots-file/actor/main.jsindex f38eb3e49..c50ba6939 100644--- a/test/e2e/playwright-robots-file/actor/main.js+++ b/test/e2e/playwright-robots-file/actor/main.js@@ -9,6 +9,8 @@ await Actor.init({});const crawler = new PlaywrightCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 10,respectRobotsTxtFile: true,});diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.jsindex 635d6f911..c78ef8241 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js@@ -11,6 +11,8 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PuppeteerCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 10,preNavigationHooks: [async ({ page }, goToOptions) => {diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.jsindex 85e4a355e..c9e1b0da7 100644--- a/test/e2e/puppeteer-store-pagination/actor/main.js+++ b/test/e2e/puppeteer-store-pagination/actor/main.js@@ -9,6 +9,8 @@ await Actor.init({});const crawler = new PuppeteerCrawler({+ // The store rate-limits the platform's shared egress IP, so crawl through a proxy.+ proxyConfiguration: await Actor.createProxyConfiguration(),maxRequestsPerCrawl: 10,preNavigationHooks: [async ({ page }, goToOptions) => {diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex be30e5480..c47615d26 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -339,6 +339,15 @@ export async function getApifyToken() {}const { token } = await fs.readJSON(authPath);++ // Newer CLI versions keep the token in the OS keyring, leaving auth.json without it.+ if (!token) {+ throw new Error(+ 'Your Apify token is not stored in auth.json (the CLI likely saved it in the OS keyring). ' ++ 'Set the "APIFY_TOKEN" environment variable, or re-run "apify login" with "APIFY_DISABLE_KEYRING=1".',+ );+ }+return token;}