Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: e2e tests fix, better logging (#1713)
Jindřich Bärfd74447438e2
diff --git a/test/e2e/automatic-persist-value/actor/apify.json b/test/e2e/automatic-persist-value/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/automatic-persist-value/actor/apify.jsonrename to test/e2e/automatic-persist-value/actor/.actor/actor.jsonindex 944ceeb3d..7f7835da4 100644--- a/test/e2e/automatic-persist-value/actor/apify.json+++ b/test/e2e/automatic-persist-value/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-automatic-persist-value","version": "0.0","buildTag": "latest",diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/autoscaling-max-tasks-per-minute/actor/apify.jsonrename to test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.jsonindex b6f64269e..e6e0c1ed5 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-autoscaling-max-tasks-per-minute","version": "0.0","buildTag": "latest",diff --git a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs b/test/e2e/autoscaling-max-tasks-per-minute/test.mjsindex 6e9351c55..3979c69e0 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs+++ b/test/e2e/autoscaling-max-tasks-per-minute/test.mjs@@ -7,6 +7,6 @@ const { stats } = await runActor(testActorDirname);await expect(stats.requestsFinished === 2, 'All requests finished');await expect(- stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 65_000,+ stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 70_000,`Ran one task per minute, took ~1 minute to complete, but no more than that (${stats.crawlerRuntimeMillis}ms)`,);diff --git a/test/e2e/cheerio-default-ts/actor/apify.json b/test/e2e/cheerio-default-ts/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/cheerio-default-ts/actor/apify.jsonrename to test/e2e/cheerio-default-ts/actor/.actor/actor.jsonindex d3e05a49e..d1802658a 100644--- a/test/e2e/cheerio-default-ts/actor/apify.json+++ b/test/e2e/cheerio-default-ts/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-default-ts","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-default/actor/apify.json b/test/e2e/cheerio-default/actor/.actor/actor.jsonsimilarity index 77%rename from test/e2e/cheerio-default/actor/apify.jsonrename to test/e2e/cheerio-default/actor/.actor/actor.jsonindex 4d5d0a673..69e8b83b6 100644--- a/test/e2e/cheerio-default/actor/apify.json+++ b/test/e2e/cheerio-default/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-default","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-enqueue-links/actor/apify.json b/test/e2e/cheerio-enqueue-links/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/cheerio-enqueue-links/actor/apify.jsonrename to test/e2e/cheerio-enqueue-links/actor/.actor/actor.jsonindex 8a288a3b4..d51768db2 100644--- a/test/e2e/cheerio-enqueue-links/actor/apify.json+++ b/test/e2e/cheerio-enqueue-links/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-enqueue-links","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/apify.json b/test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/cheerio-ignore-ssl-errors/actor/apify.jsonrename to test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.jsonindex 3faacd490..fbf791bef 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/apify.json+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-ignore-ssl-errors","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-initial-cookies/actor/apify.json b/test/e2e/cheerio-initial-cookies/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/cheerio-initial-cookies/actor/apify.jsonrename to test/e2e/cheerio-initial-cookies/actor/.actor/actor.jsonindex 98ab258fe..e3df9a5b5 100644--- a/test/e2e/cheerio-initial-cookies/actor/apify.json+++ b/test/e2e/cheerio-initial-cookies/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-initial-cookies","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-max-requests/actor/apify.json b/test/e2e/cheerio-max-requests/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/cheerio-max-requests/actor/apify.jsonrename to test/e2e/cheerio-max-requests/actor/.actor/actor.jsonindex 965ffae6d..28134cf02 100644--- a/test/e2e/cheerio-max-requests/actor/apify.json+++ b/test/e2e/cheerio-max-requests/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-max-requests","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-max-requests/actor/main.js b/test/e2e/cheerio-max-requests/actor/main.jsindex bbd6cca41..3e1344ecc 100644--- a/test/e2e/cheerio-max-requests/actor/main.js+++ b/test/e2e/cheerio-max-requests/actor/main.js@@ -15,9 +15,9 @@ await Actor.main(async () => {const { url, userData: { label } } = request;if (label === 'START') {- const links = $('.ActorStoreItem').toArray().map((item) => $(item).attr('href'));+ const links = $('a.card').toArray().map((item) => $(item).attr('href'));for (const link of links) {- const actorDetailUrl = `https://apify.com${link}`;+ const actorDetailUrl = `https://crawlee.dev${link}`;await crawler.addRequests([{url: actorDetailUrl,userData: { label: 'DETAIL' },@@ -26,21 +26,19 @@ await Actor.main(async () => {} else if (label === 'DETAIL') {const uniqueIdentifier = url.split('/').slice(-2).join('/');const title = $('header h1').text();- const description = $('header span.actor-description').text();- const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');- const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');+ const firstParagraph = $('header + p').text();+ const modifiedDate = $('.theme-last-updated time').attr('datetime');await Dataset.pushData({url,uniqueIdentifier,title,- description,- modifiedDate: new Date(Number(modifiedDate)),- runCount: Number(runCount),+ firstParagraph,+ modifiedDate,});}},});- await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);+ await crawler.run([{ url: 'https://crawlee.dev/docs/examples', userData: { label: 'START' } }]);}, mainOptions);diff --git a/test/e2e/cheerio-max-requests/test.mjs b/test/e2e/cheerio-max-requests/test.mjsindex fd8394e08..cf70bbf93 100644--- a/test/e2e/cheerio-max-requests/test.mjs+++ b/test/e2e/cheerio-max-requests/test.mjs@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);await expect(stats.requestsFinished > 10, 'All requests finished');await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items');await expect(- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),'Dataset items validation',);diff --git a/test/e2e/cheerio-page-info/actor/apify.json b/test/e2e/cheerio-page-info/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/cheerio-page-info/actor/apify.jsonrename to test/e2e/cheerio-page-info/actor/.actor/actor.jsonindex d8aa10fb9..448dc8868 100644--- a/test/e2e/cheerio-page-info/actor/apify.json+++ b/test/e2e/cheerio-page-info/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-page-info","version": "0.0","buildTag": "latest",diff --git a/test/e2e/cheerio-page-info/actor/main.js b/test/e2e/cheerio-page-info/actor/main.jsindex 62f6e4c4e..191217c29 100644--- a/test/e2e/cheerio-page-info/actor/main.js+++ b/test/e2e/cheerio-page-info/actor/main.js@@ -12,7 +12,7 @@ const router = createCheerioRouter();router.addHandler('START', async ({ enqueueLinks }) => {await enqueueLinks({label: 'DETAIL',- globs: ['https://apify.com/apify/web-scraper'],+ globs: ['**/examples/accept-user-input'],});});@@ -21,17 +21,15 @@ router.addHandler('DETAIL', async ({ request, $ }) => {const uniqueIdentifier = url.split('/').slice(-2).join('/');const title = $('header h1').text();- const description = $('header span.actor-description').text();- const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');- const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');+ const firstParagraph = $('header + p').text();+ const modifiedDate = $('.theme-last-updated time').attr('datetime');await Dataset.pushData({url,uniqueIdentifier,title,- description,- modifiedDate: new Date(Number(modifiedDate)),- runCount: Number(runCount),+ firstParagraph,+ modifiedDate,});});@@ -40,5 +38,5 @@ await Actor.main(async () => {requestHandler: router,});- await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);+ await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);}, mainOptions);diff --git a/test/e2e/cheerio-page-info/test.mjs b/test/e2e/cheerio-page-info/test.mjsindex e051e3b84..6ed16a4f6 100644--- a/test/e2e/cheerio-page-info/test.mjs+++ b/test/e2e/cheerio-page-info/test.mjs@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);await expect(stats.requestsFinished === 2, 'All requests finished');await expect(datasetItems.length === 1, 'Number of dataset items');await expect(- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),'Dataset items validation',);diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/cheerio-throw-on-ssl-errors/actor/apify.jsonrename to test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.jsonindex 71bf83458..051de1117 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-cheerio-throw-on-ssl-errors","version": "0.0","buildTag": "latest",diff --git a/test/e2e/input-json5/actor/apify.json b/test/e2e/input-json5/actor/.actor/actor.jsonsimilarity index 77%rename from test/e2e/input-json5/actor/apify.jsonrename to test/e2e/input-json5/actor/.actor/actor.jsonindex aa48ca695..c3a5b0ca6 100644--- a/test/e2e/input-json5/actor/apify.json+++ b/test/e2e/input-json5/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-input-json5","version": "0.0","buildTag": "latest",diff --git a/test/e2e/jsdom-default-ts/actor/apify.json b/test/e2e/jsdom-default-ts/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/jsdom-default-ts/actor/apify.jsonrename to test/e2e/jsdom-default-ts/actor/.actor/actor.jsonindex dfa4a1d45..976cd31c4 100644--- a/test/e2e/jsdom-default-ts/actor/apify.json+++ b/test/e2e/jsdom-default-ts/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-jsdom-default-ts","version": "0.0","buildTag": "latest",diff --git a/test/e2e/jsdom-react-ts/actor/apify.json b/test/e2e/jsdom-react-ts/actor/.actor/actor.jsonsimilarity index 77%rename from test/e2e/jsdom-react-ts/actor/apify.jsonrename to test/e2e/jsdom-react-ts/actor/.actor/actor.jsonindex 834a0018b..9fe495397 100644--- a/test/e2e/jsdom-react-ts/actor/apify.json+++ b/test/e2e/jsdom-react-ts/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-jsdom-react-ts","version": "0.0","buildTag": "latest",diff --git a/test/e2e/migration/actor/apify.json b/test/e2e/migration/actor/.actor/actor.jsonsimilarity index 76%rename from test/e2e/migration/actor/apify.jsonrename to test/e2e/migration/actor/.actor/actor.jsonindex f65813559..29134d82e 100644--- a/test/e2e/migration/actor/apify.json+++ b/test/e2e/migration/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-migration","version": "0.0","buildTag": "latest",diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.jsindex 9aa1963f1..f06fc8ece 100644--- a/test/e2e/migration/actor/main.js+++ b/test/e2e/migration/actor/main.js@@ -37,7 +37,7 @@ if (workerData !== '#actor') {maxRequestsPerCrawl: 5,async requestHandler({ enqueueLinks, request }) {const { url } = request;- await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });+ await enqueueLinks();await Dataset.pushData({ url });@@ -48,6 +48,6 @@ if (workerData !== '#actor') {// eslint-disable-next-line no-underscore-dangleConfiguration.getGlobalConfig().getStorageClient().__purged = false;- await crawler.run(['https://apify.com']);+ await crawler.run(['https://crawlee.dev']);}, mainOptions);}diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/apify.json b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.jsonsimilarity index 82%rename from test/e2e/playwright-chromium-experimental-containers/actor/apify.jsonrename to test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.jsonindex 7f43847ae..0be68bf20 100644--- a/test/e2e/playwright-chromium-experimental-containers/actor/apify.json+++ b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-playwright-chromium-experimental-containers","version": "0.0","buildTag": "latest",diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfileindex de06663ad..58d737169 100644--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/playwright-default/actor/apify.json b/test/e2e/playwright-default/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/playwright-default/actor/apify.jsonrename to test/e2e/playwright-default/actor/.actor/actor.jsonindex ddddf0cba..9ed1dfad0 100644--- a/test/e2e/playwright-default/actor/apify.json+++ b/test/e2e/playwright-default/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-playwright-default","version": "0.0","buildTag": "latest",diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfileindex de06663ad..58d737169 100644--- a/test/e2e/playwright-default/actor/Dockerfile+++ b/test/e2e/playwright-default/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.jsindex 5592a895b..5a725fbad 100644--- a/test/e2e/playwright-default/actor/main.js+++ b/test/e2e/playwright-default/actor/main.js@@ -16,9 +16,11 @@ await Actor.main(async () => {const { url } = request;const pageTitle = await page.title();await Dataset.pushData({ url, pageTitle });- await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });+ await enqueueLinks({+ globs: ['**/3.0/examples/*'],+ });},});- await crawler.run(['https://apify.com']);+ await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);}, mainOptions);diff --git a/test/e2e/playwright-default/test.mjs b/test/e2e/playwright-default/test.mjsindex 7b0d5153c..b82824ae1 100644--- a/test/e2e/playwright-default/test.mjs+++ b/test/e2e/playwright-default/test.mjs@@ -5,6 +5,6 @@ await initialize(testActorDirname);const { stats, datasetItems } = await runActor(testActorDirname, 16384);-await expect(stats.requestsFinished > 50, 'All requests finished');-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');+await expect(stats.requestsFinished > 15, 'All requests finished');+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');diff --git a/test/e2e/playwright-enqueue-links/actor/apify.json b/test/e2e/playwright-enqueue-links/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/playwright-enqueue-links/actor/apify.jsonrename to test/e2e/playwright-enqueue-links/actor/.actor/actor.jsonindex e36af37c1..7294db34b 100644--- a/test/e2e/playwright-enqueue-links/actor/apify.json+++ b/test/e2e/playwright-enqueue-links/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-playwright-enqueue-links","version": "0.0","buildTag": "latest",diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfileindex de06663ad..58d737169 100644--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/apify.json b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.jsonsimilarity index 82%rename from test/e2e/playwright-firefox-experimental-containers/actor/apify.jsonrename to test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.jsonindex 8bc7a689e..d1bf754a5 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/apify.json+++ b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-playwright-firefox-experimental-containers","version": "0.0","buildTag": "latest",diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfileindex 8a1c0fff0..af65b94eb 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/playwright-initial-cookies/actor/apify.json b/test/e2e/playwright-initial-cookies/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/playwright-initial-cookies/actor/apify.jsonrename to test/e2e/playwright-initial-cookies/actor/.actor/actor.jsonindex ca72b5fff..0404c8af3 100644--- a/test/e2e/playwright-initial-cookies/actor/apify.json+++ b/test/e2e/playwright-initial-cookies/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-playwright-initial-cookies","version": "0.0","buildTag": "latest",diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfileindex de06663ad..58d737169 100644--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/proxy-rotation/actor/apify.json b/test/e2e/proxy-rotation/actor/.actor/actor.jsonsimilarity index 77%rename from test/e2e/proxy-rotation/actor/apify.jsonrename to test/e2e/proxy-rotation/actor/.actor/actor.jsonindex eb8ac5e01..4af4d0489 100644--- a/test/e2e/proxy-rotation/actor/apify.json+++ b/test/e2e/proxy-rotation/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-proxy-rotation","version": "0.0","buildTag": "latest",diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/proxy-rotation/actor/Dockerfile+++ b/test/e2e/proxy-rotation/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-default/actor/apify.json b/test/e2e/puppeteer-default/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/puppeteer-default/actor/apify.jsonrename to test/e2e/puppeteer-default/actor/.actor/actor.jsonindex f57a60ea8..f2fa978c5 100644--- a/test/e2e/puppeteer-default/actor/apify.json+++ b/test/e2e/puppeteer-default/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-default","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-default/actor/Dockerfile+++ b/test/e2e/puppeteer-default/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.jsindex 291249bda..699239d1a 100644--- a/test/e2e/puppeteer-default/actor/main.js+++ b/test/e2e/puppeteer-default/actor/main.js@@ -16,9 +16,11 @@ await Actor.main(async () => {const { url } = request;const pageTitle = await page.title();await Dataset.pushData({ url, pageTitle });- await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });+ await enqueueLinks({+ globs: ['**/3.0/examples/*'],+ });},});- await crawler.run(['https://apify.com']);+ await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);}, mainOptions);diff --git a/test/e2e/puppeteer-default/test.mjs b/test/e2e/puppeteer-default/test.mjsindex 7b0d5153c..b82824ae1 100644--- a/test/e2e/puppeteer-default/test.mjs+++ b/test/e2e/puppeteer-default/test.mjs@@ -5,6 +5,6 @@ await initialize(testActorDirname);const { stats, datasetItems } = await runActor(testActorDirname, 16384);-await expect(stats.requestsFinished > 50, 'All requests finished');-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');+await expect(stats.requestsFinished > 15, 'All requests finished');+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');diff --git a/test/e2e/puppeteer-enqueue-links/actor/apify.json b/test/e2e/puppeteer-enqueue-links/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/puppeteer-enqueue-links/actor/apify.jsonrename to test/e2e/puppeteer-enqueue-links/actor/.actor/actor.jsonindex 7dd9d68c5..7beecb430 100644--- a/test/e2e/puppeteer-enqueue-links/actor/apify.json+++ b/test/e2e/puppeteer-enqueue-links/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-enqueue-links","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/puppeteer-ignore-ssl-errors/actor/apify.jsonrename to test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.jsonindex 049cfe03d..abbeda5d9 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-ignore-ssl-errors","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-initial-cookies/actor/apify.json b/test/e2e/puppeteer-initial-cookies/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/puppeteer-initial-cookies/actor/apify.jsonrename to test/e2e/puppeteer-initial-cookies/actor/.actor/actor.jsonindex aa2946018..328c5bb8a 100644--- a/test/e2e/puppeteer-initial-cookies/actor/apify.json+++ b/test/e2e/puppeteer-initial-cookies/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-initial-cookies","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-page-info/actor/apify.json b/test/e2e/puppeteer-page-info/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/puppeteer-page-info/actor/apify.jsonrename to test/e2e/puppeteer-page-info/actor/.actor/actor.jsonindex e136f456e..285cbce73 100644--- a/test/e2e/puppeteer-page-info/actor/apify.json+++ b/test/e2e/puppeteer-page-info/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-page-info","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-page-info/actor/Dockerfile+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.jsindex ee853ff0e..e851d4569 100644--- a/test/e2e/puppeteer-page-info/actor/main.js+++ b/test/e2e/puppeteer-page-info/actor/main.js@@ -17,7 +17,7 @@ await Actor.main(async () => {if (label === 'START') {await enqueueLinks({- globs: [{ glob: 'https://apify.com/apify/web-scraper', userData: { label: 'DETAIL' } }],+ globs: ['**/examples/accept-user-input'], userData: { label: 'DETAIL' },});}@@ -27,28 +27,22 @@ await Actor.main(async () => {const uniqueIdentifier = url.split('/').slice(-2).join('/');const titleP = page.$eval('header h1', ((el) => el.textContent));- const descriptionP = page.$eval('header span.actor-description', ((el) => el.textContent));- const modifiedTimestampP = page.$eval('ul.ActorHeader-stats time', (el) => el.getAttribute('datetime'));- const runCountTextP = page.$eval('ul.ActorHeader-stats > li:nth-of-type(3)', ((el) => el.textContent));+ const firstParagraphP = page.$eval('header + p', ((el) => el.textContent));+ const modifiedDateP = page.$eval('.theme-last-updated time', (el) => el.getAttribute('datetime'));const [title,description,- modifiedTimestamp,- runCountText,+ modifiedDate,] = await Promise.all([titleP,- descriptionP,- modifiedTimestampP,- runCountTextP,+ firstParagraphP,+ modifiedDateP,]);- const modifiedDate = new Date(Number(modifiedTimestamp));- const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));-- await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate });}},});- await crawler.run([{ url: 'https://apify.com/store', userData: { label: 'START' } }]);+ await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);}, mainOptions);diff --git a/test/e2e/puppeteer-page-info/test.mjs b/test/e2e/puppeteer-page-info/test.mjsindex 3f9f005d1..06d47068c 100644--- a/test/e2e/puppeteer-page-info/test.mjs+++ b/test/e2e/puppeteer-page-info/test.mjs@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname, 16384);await expect(stats.requestsFinished === 2, 'All requests finished');await expect(datasetItems.length === 1, 'Number of dataset items');await expect(- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate']),'Dataset items validation',);diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/apify.json b/test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/puppeteer-store-pagination-jquery/actor/apify.jsonrename to test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.jsonindex 238376e78..8d6941f71 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/apify.json+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-store-pagination-jquery","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.jsindex f18f8b5dc..8d641cd52 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js@@ -26,7 +26,7 @@ await Actor.main(async () => {await page.waitForNetworkIdle();// Enqueue all loaded linksawait enqueueLinks({- selector: 'a.ActorStoreItem',+ selector: '[data-test="actorCard"] > a',globs: [{ glob: 'https://apify.com/*/*', userData: { label: 'DETAIL' } }],});log.info(`Enqueued actors for page ${pageNo}`);diff --git a/test/e2e/puppeteer-store-pagination/actor/apify.json b/test/e2e/puppeteer-store-pagination/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/puppeteer-store-pagination/actor/apify.jsonrename to test/e2e/puppeteer-store-pagination/actor/.actor/actor.jsonindex d87f9cd56..0fd41a53d 100644--- a/test/e2e/puppeteer-store-pagination/actor/apify.json+++ b/test/e2e/puppeteer-store-pagination/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-store-pagination","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.jsindex 9357d65d7..86726517a 100644--- a/test/e2e/puppeteer-store-pagination/actor/main.js+++ b/test/e2e/puppeteer-store-pagination/actor/main.js@@ -21,7 +21,7 @@ crawler.router.addHandler('START', async ({ log, enqueueLinks, page }) => {await page.waitForNetworkIdle();// Enqueue all loaded linksawait enqueueLinks({- selector: 'a.ActorStoreItem',+ selector: '[data-test="actorCard"] > a',label: 'DETAIL',globs: ['https://apify.com/*/*'],});diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.jsonrename to test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.jsonindex 8c375a749..224de98d2 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.json+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-puppeteer-throw-on-ssl-errors","version": "0.0","buildTag": "latest",diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfileindex 7c7d30b82..de6b194d0 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile@@ -12,7 +12,7 @@ RUN rm -r node_modulesCOPY --from=builder /node_modules ./node_modulesCOPY --from=builder /packages ./packagesCOPY --from=builder /package*.json ./-COPY /apify.json ./+COPY /.actor ./.actorCOPY /main.js ./RUN echo "Installed NPM packages:" \diff --git a/test/e2e/request-queue-zero-concurrency/actor/apify.json b/test/e2e/request-queue-zero-concurrency/actor/.actor/actor.jsonsimilarity index 80%rename from test/e2e/request-queue-zero-concurrency/actor/apify.jsonrename to test/e2e/request-queue-zero-concurrency/actor/.actor/actor.jsonindex ccfd2bacd..959ec7eaa 100644--- a/test/e2e/request-queue-zero-concurrency/actor/apify.json+++ b/test/e2e/request-queue-zero-concurrency/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-request-queue-zero-concurrency","version": "0.0","buildTag": "latest",diff --git a/test/e2e/request-skip-navigation/actor/apify.json b/test/e2e/request-skip-navigation/actor/.actor/actor.jsonsimilarity index 79%rename from test/e2e/request-skip-navigation/actor/apify.jsonrename to test/e2e/request-skip-navigation/actor/.actor/actor.jsonindex 391f5a195..0f06a7057 100644--- a/test/e2e/request-skip-navigation/actor/apify.json+++ b/test/e2e/request-skip-navigation/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-request-skip-navigation","version": "0.0","buildTag": "latest",diff --git a/test/e2e/session-rotation/actor/apify.json b/test/e2e/session-rotation/actor/.actor/actor.jsonsimilarity index 78%rename from test/e2e/session-rotation/actor/apify.jsonrename to test/e2e/session-rotation/actor/.actor/actor.jsonindex bac48c79c..7308531ed 100644--- a/test/e2e/session-rotation/actor/apify.json+++ b/test/e2e/session-rotation/actor/.actor/actor.json@@ -1,4 +1,5 @@{+ "actorSpecification": 1,"name": "test-session-rotation","version": "0.0","buildTag": "latest",diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.jsindex 06a711315..107dd47c0 100644--- a/test/e2e/session-rotation/actor/main.js+++ b/test/e2e/session-rotation/actor/main.js@@ -17,7 +17,7 @@ await Actor.main(async () => {},requestHandler: async ({ session }) => {const { id, usageCount, errorScore } = session;- Actor.pushData({ id, usageCount, errorScore });+ await Actor.pushData({ id, usageCount, errorScore });throw new Error('retry');},});diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 1603168b9..85f9aa29f 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -4,13 +4,17 @@ import { existsSync } from 'node:fs';import { readdir, readFile } from 'node:fs/promises';import { homedir } from 'node:os';import { setTimeout } from 'node:timers/promises';-import { execSync } from 'node:child_process';+import { execSync as execSyncOriginal } from 'node:child_process';import { got } from 'got';import fs from 'fs-extra';import { Actor } from 'apify';// eslint-disable-next-line import/no-relative-packagesimport { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs';+function execSync(command, options) {+ return execSyncOriginal(command, { ...options, encoding: 'utf-8' });+}+/*** @param {string} name*/