Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Small fixies according to PR
Marek Trunkat347339a72d59
diff --git a/src/utils.js b/src/utils.jsindex dd15c419e..25dfe1e2e 100644--- a/src/utils.js+++ b/src/utils.js@@ -122,14 +122,6 @@ export const addCharsetToContentType = (contentType) => {return contentTypeParser.format(parsed);};-/**- * Returns promise that resolves to true if the code is running in Docker container.- * See https://github.com/sindresorhus/is-docker- *- * Param forceReset is just internal for unit tests.- *- * @return Promise- */let isDockerPromise;const createIsDockerPromise = () => {const promise1 = Promise@@ -146,6 +138,15 @@ const createIsDockerPromise = () => {.all([promise1, promise2]).then(([result1, result2]) => result1 || result2);};++/**+ * Returns promise that resolves to true if the code is running in Docker container.+ * See https://github.com/sindresorhus/is-docker+ *+ * Param forceReset is just internal for unit tests.+ *+ * @return Promise+ */export const isDocker = (forceReset) => {if (!isDockerPromise || forceReset) isDockerPromise = createIsDockerPromise();diff --git a/test/actor.js b/test/actor.jsindex d86331e4d..9bc5311b7 100644--- a/test/actor.js+++ b/test/actor.js@@ -1238,7 +1238,7 @@ describe('Apify.call()', () => {});describe('Apify.getMemoryInfo()', () => {- it('works in docker container', () => {+ it('works outside the container', () => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);@@ -1271,6 +1271,36 @@ describe('Apify.getMemoryInfo()', () => {});});+ it('works inside the container', () => {+ const utilsMock = sinon.mock(utils);++ utilsMock+ .expects('isDocker')+ .once()+ .returns(Promise.resolve(true));++ sinon+ .stub(fs, 'readFile')+ .callsFake((path, callback) => {+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');+ else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');+ else throw new Error('Invalid path');+ });++ return Apify+ .getMemoryInfo()+ .then((data) => {+ expect(data).to.be.eql({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,+ });++ utilsMock.restore();+ fs.readFile.restore();+ });+ });+it('works with callback', (done) => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);@@ -1302,34 +1332,4 @@ describe('Apify.getMemoryInfo()', () => {done();});});-- it('works outside the container', () => {- const utilsMock = sinon.mock(utils);-- utilsMock- .expects('isDocker')- .once()- .returns(Promise.resolve(true));-- sinon- .stub(fs, 'readFile')- .callsFake((path, callback) => {- if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') callback(null, '333');- else if (path === '/sys/fs/cgroup/memory/memory.usage_in_bytes') callback(null, '111');- else throw new Error('Invalid path');- });-- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).to.be.eql({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });-- utilsMock.restore();- fs.readFile.restore();- });- });});