Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Do not add charset in setValue
Ondra Urban1ae246be7896
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3b6d1353c..c349cd7c4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -170,6 +170,8 @@ Full list of changes:Use `persistRequestsKey`.- Removed the `The function passed to Apify.main() threw an exception` error message,because it was confusing to users.+- Removed automatic injection of `charset=utf-8` in `keyValueStore.setValue()`+ to the `contentType` option.0.22.0 / 2020/12/07====================diff --git a/src/storages/key_value_store.js b/src/storages/key_value_store.jsindex fe421c0e7..a27f0c963 100644--- a/src/storages/key_value_store.js+++ b/src/storages/key_value_store.js@@ -4,7 +4,6 @@ import { jsonStringifyExtended } from 'apify-shared/utilities';import ow, { ArgumentError } from 'ow';import { APIFY_API_BASE_URL } from '../constants';import StorageManager from './storage_manager';-import { addCharsetToContentType } from '../utils';import log from '../utils_log';/**@@ -15,7 +14,7 @@ import log from '../utils_log';export const maybeStringify = (value, options) => {// If contentType is missing, value will be stringified to JSONif (options.contentType === null || options.contentType === undefined) {- options.contentType = 'application/json';+ options.contentType = 'application/json; charset=utf-8';try {// Format JSON to simplify debugging, the overheads with compression is negligible@@ -223,13 +222,14 @@ export class KeyValueStore {// In this case delete the record.if (value === null) return this.client.deleteRecord(key);+ // TODO the function mutates optionsCopy, but is also used in actor.js+ // Remove the mutation when actor.js usages are removed.value = maybeStringify(value, optionsCopy);- // Keep this code in main scope so that simple errors are thrown rather than rejected promise.return this.client.setRecord({key,value,- contentType: addCharsetToContentType(optionsCopy.contentType),+ contentType: optionsCopy.contentType,});}diff --git a/test/storages/key_value_store.test.js b/test/storages/key_value_store.test.jsindex 35d563714..21a783e8d 100644--- a/test/storages/key_value_store.test.js+++ b/test/storages/key_value_store.test.js@@ -201,7 +201,7 @@ describe('KeyValueStore remote', () => {.spyOn(store.client, 'setRecord').mockResolvedValueOnce(null);- await store.setValue('key-1', 'xxxx', { contentType: 'text/plain' });+ await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' });expect(mockSetRecord).toHaveBeenCalledTimes(1);expect(mockSetRecord).toHaveBeenCalledWith({