Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Do not add charset in setValue

Ondra Urban1ae246be7896
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3b6d1353c..c349cd7c4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -170,6 +170,8 @@ Full list of changes:
Use `persistRequestsKey`.
- Removed the `The function passed to Apify.main() threw an exception` error message,
because it was confusing to users.
+- Removed automatic injection of `charset=utf-8` in `keyValueStore.setValue()`
+ to the `contentType` option.
0.22.0 / 2020/12/07
====================
diff --git a/src/storages/key_value_store.js b/src/storages/key_value_store.js
index fe421c0e7..a27f0c963 100644
--- a/src/storages/key_value_store.js
+++ b/src/storages/key_value_store.js
@@ -4,7 +4,6 @@ import { jsonStringifyExtended } from 'apify-shared/utilities';
import ow, { ArgumentError } from 'ow';
import { APIFY_API_BASE_URL } from '../constants';
import StorageManager from './storage_manager';
-import { addCharsetToContentType } from '../utils';
import log from '../utils_log';
/**
@@ -15,7 +14,7 @@ import log from '../utils_log';
export const maybeStringify = (value, options) => {
// If contentType is missing, value will be stringified to JSON
if (options.contentType === null || options.contentType === undefined) {
- options.contentType = 'application/json';
+ options.contentType = 'application/json; charset=utf-8';
try {
// Format JSON to simplify debugging, the overheads with compression is negligible
@@ -223,13 +222,14 @@ export class KeyValueStore {
// In this case delete the record.
if (value === null) return this.client.deleteRecord(key);
+ // TODO the function mutates optionsCopy, but is also used in actor.js
+ // Remove the mutation when actor.js usages are removed.
value = maybeStringify(value, optionsCopy);
- // Keep this code in main scope so that simple errors are thrown rather than rejected promise.
return this.client.setRecord({
key,
value,
- contentType: addCharsetToContentType(optionsCopy.contentType),
+ contentType: optionsCopy.contentType,
});
}
diff --git a/test/storages/key_value_store.test.js b/test/storages/key_value_store.test.js
index 35d563714..21a783e8d 100644
--- a/test/storages/key_value_store.test.js
+++ b/test/storages/key_value_store.test.js
@@ -201,7 +201,7 @@ describe('KeyValueStore remote', () => {
.spyOn(store.client, 'setRecord')
.mockResolvedValueOnce(null);
- await store.setValue('key-1', 'xxxx', { contentType: 'text/plain' });
+ await store.setValue('key-1', 'xxxx', { contentType: 'text/plain; charset=utf-8' });
expect(mockSetRecord).toHaveBeenCalledTimes(1);
expect(mockSetRecord).toHaveBeenCalledWith({