Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: use self-hosted httpbin instance in e2e tests (#2967)
Jindřich Bär52531cb8b8c8
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.ymlindex 34429940f..2f1295520 100644--- a/.github/workflows/test-e2e.yml+++ b/.github/workflows/test-e2e.yml@@ -70,3 +70,4 @@ jobs:run: yarn test:e2eenv:STORAGE_IMPLEMENTATION: ${{ matrix.storage }}+ APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}diff --git a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts b/test/e2e/cheerio-curl-impersonate-ts/actor/main.tsindex 0f6b4fdcb..9ccb08150 100644--- a/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts+++ b/test/e2e/cheerio-curl-impersonate-ts/actor/main.ts@@ -163,19 +163,31 @@ class CurlImpersonateHttpClient implements BaseHttpClient {}}+function getHttpBinUrl(path: string): string {+ let url: URL;+ if (process.env.APIFY_HTTPBIN_TOKEN) {+ url = new URL(path, 'https://httpbin.apify.actor');+ url.searchParams.set('token', process.env.APIFY_HTTPBIN_TOKEN);+ } else {+ url = new URL(path, 'https://httpbin.org');+ }++ return url.href;+}+const crawler = new CheerioCrawler({async requestHandler(context) {const { body: text } = await context.sendRequest({- url: 'https://httpbin.org/uuid',+ url: getHttpBinUrl('/uuid'),});const { body: json } = await context.sendRequest({- url: 'https://httpbin.org/uuid',+ url: getHttpBinUrl('/uuid'),responseType: 'json',});const { body: ua } = await context.sendRequest<Dictionary>({- url: 'https://httpbin.org/user-agent',+ url: getHttpBinUrl('/user-agent'),responseType: 'json',});@@ -190,6 +202,6 @@ const crawler = new CheerioCrawler({httpClient: new CurlImpersonateHttpClient({ impersonate: 'chrome-116' }),});-await crawler.run(['https://httpbin.org/']);+await crawler.run([getHttpBinUrl('/')]);await Actor.exit({ exit: Actor.isAtHome() });diff --git a/test/e2e/cheerio-impit-ts/actor/main.ts b/test/e2e/cheerio-impit-ts/actor/main.tsindex 496c39c06..8a1b32a19 100644--- a/test/e2e/cheerio-impit-ts/actor/main.ts+++ b/test/e2e/cheerio-impit-ts/actor/main.ts@@ -10,19 +10,31 @@ if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {await Actor.init();}+function getHttpBinUrl(path: string): string {+ let url: URL;+ if (process.env.APIFY_HTTPBIN_TOKEN) {+ url = new URL(path, 'https://httpbin.apify.actor');+ url.searchParams.set('token', process.env.APIFY_HTTPBIN_TOKEN);+ } else {+ url = new URL(path, 'https://httpbin.org');+ }++ return url.href;+}+const crawler = new CheerioCrawler({async requestHandler(context) {const { body: text } = await context.sendRequest({- url: 'https://httpbin.org/uuid',+ url: getHttpBinUrl('/uuid'),});const { body: json } = await context.sendRequest({- url: 'https://httpbin.org/uuid',+ url: getHttpBinUrl('/uuid'),responseType: 'json',});const { body: ua } = await context.sendRequest<Dictionary>({- url: 'https://httpbin.org/user-agent',+ url: getHttpBinUrl('/user-agent'),responseType: 'json',});@@ -37,6 +49,6 @@ const crawler = new CheerioCrawler({httpClient: new ImpitHttpClient({ browser: Browser.Firefox }),});-await crawler.run(['https://httpbin.org/']);+await crawler.run([getHttpBinUrl('/')]);await Actor.exit({ exit: Actor.isAtHome() });