Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding charset=utf-8 when storing the record to key value store
Marek Trunkata316b10dd76d
diff --git a/package.json b/package.jsonindex 6c8d17b87..b0aa266bc 100644--- a/package.json+++ b/package.json@@ -41,11 +41,12 @@},"dependencies": {"apify-client": ">=0.1.13",- "underscore": "^1.8.3",+ "bluebird": "^3.5.0",+ "content-type": "^1.0.3","portastic": "^1.0.1",+ "rimraf": "^2.6.1","tmp": "^0.0.33",- "bluebird": "^3.5.0",- "rimraf": "^2.6.1"+ "underscore": "^1.8.3"},"devDependencies": {"babel-cli": "^6.6.5",diff --git a/src/actor.js b/src/actor.jsindex 7f12f8363..81573f5e4 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,10 +1,11 @@import fs from 'fs';import path from 'path';import _ from 'underscore';+import contentTypeParser from 'content-type';import Promise from 'bluebird';import { checkParamOrThrow } from 'apify-client/build/utils';import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES } from './constants';-import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './utils';+import { getPromisePrototype, newPromise, nodeifyPromise, newClient, addCharsetToContentType } from './utils';/* global process, Buffer */@@ -63,7 +64,8 @@ export const getValue = (key, callback = null) => {if (devDir) {// We're emulating KV store locally in a directory to simplify development- const contentType = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_CONTENT_TYPE] || 'application/json';+ const devContentType = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_CONTENT_TYPE] || 'application/json; charset=utf-8';+ const contentType = contentTypeParser.parse(devContentType).type;const dirPath = path.resolve(devDir);let filePath;@@ -91,7 +93,7 @@ export const getValue = (key, callback = null) => {.then((data) => {// Parse file according to the content typeif (data !== null) {- if (contentType === 'application/json' || contentType === 'application/json; charset=utf-8') {+ if (contentType === 'application/json') {try {data = JSON.parse(data.toString('utf8'));} catch (e) {@@ -210,7 +212,7 @@ export const setValue = (key, value, options, callback = null) => {promise: promisePrototype,key,body: value,- contentType: options.contentType,+ contentType: addCharsetToContentType(options.contentType),});}} else {@@ -367,7 +369,7 @@ export const call = (opts) => {const { body, contentType } = input;checkParamOrThrow(body, 'body', 'Buffer | String');checkParamOrThrow(contentType, 'contentType', 'String');- runActOpts.contentType = contentType;+ runActOpts.contentType = addCharsetToContentType(contentType);runActOpts.body = body;}diff --git a/src/utils.js b/src/utils.jsindex 38921a9b2..2decd9a04 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,4 +1,5 @@import urlModule from 'url';+import contentTypeParser from 'content-type';import ApifyClient from 'apify-client';import { ENV_VARS } from './constants';@@ -138,3 +139,21 @@ export const redactParsedUrl = (parsedUrl, passwordReplacement = '<redacted>') =}return `${p.protocol}//${auth || ''}${auth ? '@' : ''}${p.host}${p.path || ''}${p.hash || ''}`;};++/**+ * Adds charset=utf-8 to given content type if this parameter is missing.+ *+ * @param contentType+ * @return {string}+ */+export const addCharsetToContentType = (contentType) => {+ if (!contentType) return contentType;++ const parsed = contentTypeParser.parse(contentType);++ if (parsed.parameters.charset) return contentType;++ parsed.parameters.charset = 'utf-8';++ return contentTypeParser.format(parsed);+};diff --git a/test/actor.js b/test/actor.jsindex be0d1b317..e786fe22e 100644--- a/test/actor.js+++ b/test/actor.js@@ -703,7 +703,7 @@ describe('Apify.setValue()', () => {promise: Promise,key,body: JSON.stringify(value, null, 2),- contentType: 'application/json',+ contentType: 'application/json; charset=utf-8',}).returns(Promise.resolve(null));@@ -738,7 +738,7 @@ describe('Apify.setValue()', () => {promise: Promise,key,body: value,- contentType,+ contentType: `${contentType}; charset=utf-8`,}).returns(Promise.resolve(null));@@ -759,7 +759,7 @@ describe('Apify.setValue()', () => {const storeId = 'mystore3';const key = 'mykey2';const value = Buffer.from('some text value');- const contentType = 'text/plain';+ const contentType = 'text/plain; charset=something';process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = storeId;@@ -935,7 +935,7 @@ describe('Apify.call()', () => {const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('runAct')- .withExactArgs({ token, actId, contentType: input.contentType, body: input.body, build })+ .withExactArgs({ token, actId, contentType: `${input.contentType}; charset=utf-8`, body: input.body, build }).once().returns(Promise.resolve(runningRun));actsMock.expects('getRun')diff --git a/test/utils.js b/test/utils.jsindex ec31dcc21..8d7605471 100644--- a/test/utils.js+++ b/test/utils.js@@ -134,3 +134,14 @@ describe('utils.redactUrl()', () => {.to.eql('ftp://example.com/');});});++describe('utils.addCharsetToContentType()', () => {+ it('works', () => {+ expect(utils.addCharsetToContentType('application/json; charset=something')).to.eql('application/json; charset=something');+ expect(utils.addCharsetToContentType('application/json; foo=bar; charset=something')).to.eql('application/json; foo=bar; charset=something');+ expect(utils.addCharsetToContentType('application/json; foo=bar')).to.eql('application/json; charset=utf-8; foo=bar');+ expect(utils.addCharsetToContentType('application/json')).to.eql('application/json; charset=utf-8');+ expect(utils.addCharsetToContentType(null)).to.eql(null);+ expect(utils.addCharsetToContentType(undefined)).to.eql(undefined);+ });+});