Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding test for content type charset auto setting
Marek Trunkat5f333909fa24
diff --git a/test/actor.js b/test/actor.jsindex e786fe22e..6744e33c8 100644--- a/test/actor.js+++ b/test/actor.js@@ -719,12 +719,46 @@ describe('Apify.setValue()', () => {});});+ it('correctly adds charset to content type', () => {+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';+ const storeId = 'mystore3';+ const key = 'mykey2';+ const value = 'some string value';++ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;++ Apify.setPromisesDependency(Promise);++ const mock = sinon.mock(Apify.client.keyValueStores);+ mock.expects('putRecord')+ .once()+ .withArgs({+ storeId,+ promise: Promise,+ key,+ body: value,+ contentType: 'text/plain; charset=utf-8; foo=bar',+ })+ .returns(Promise.resolve(null));++ return Promise.resolve()+ .then(() => {+ return Apify.setValue(key, value, { contentType: 'text/plain; foo=bar' });+ })+ .then(() => {+ mock.verify();+ })+ .finally(() => {+ mock.restore();+ });+ });+it('correctly stores raw string values', () => {process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';const storeId = 'mystore3';const key = 'mykey2';const value = 'some string value';- const contentType = 'text/plain';+ const contentType = 'text/plain; charset=utf-8';process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;@@ -738,7 +772,7 @@ describe('Apify.setValue()', () => {promise: Promise,key,body: value,- contentType: `${contentType}; charset=utf-8`,+ contentType,}).returns(Promise.resolve(null));