Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixes from PR and broken tests
Marek Trunkatb57b06128e10
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex a513ccad8..6232c9956 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -626,7 +626,11 @@ export const openKeyValueStore = (storeIdOrName, options = {}) => {* @name getValue* @function*/-export const getValue = async key => openKeyValueStore().then(store => store.getValue(key));+export const getValue = async (key) => {+ const store = await openKeyValueStore();++ return store.getValue(key);+};/*** Stores or deletes a value in the default {@link KeyValueStore} associated with the current actor run.@@ -667,7 +671,11 @@ export const getValue = async key => openKeyValueStore().then(store => store.get* @name setValue* @function*/-export const setValue = async (key, value, options) => openKeyValueStore().then(store => store.setValue(key, value, options));+export const setValue = async (key, value, options) => {+ const store = await openKeyValueStore();++ return store.setValue(key, value, options);+};/*** Gets the actor input value from the default {@link KeyValueStore} associated with the current actor run.diff --git a/test/utils_social.js b/test/utils_social.jsindex d2d745fe9..259fd66c0 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -1,8 +1,11 @@import _ from 'underscore';-import { expect } from 'chai';+import chai, { expect } from 'chai';+import chaiAsPromised from 'chai-as-promised';// import * as utils from '../build/utils';import Apify from '../build/index';+chai.use(chaiAsPromised);+const { social } = Apify.utils;/* global describe, it */@@ -39,7 +42,7 @@ describe('utils.social', () => {testEmailsFromText(`this'is'also'valid'[email protected][email protected] \n\nnot @ an.email.com@also.not.an.email`, [@@ -127,10 +130,10 @@ describe('utils.social', () => {it('extracts phones correctly', () => {testPhonesFromText(`+420775123456 +420775123456-++420 775 123 456-- 775123456 775123456 \n\n++ 775123456 775123456 \n\n004207751234561234567 1234567890+44 7911 123456@@ -219,7 +222,7 @@ describe('utils.social', () => {000-0000000 0000000.0000-+00000000000000000(000)0000000@@ -334,26 +337,26 @@ describe('utils.social', () => {expect(social.parseHandlesFromHtml(`<html><head>- <title>Bla</title>+ <title>Bla</title></head><a>- <p>[email protected]</p>- [email protected] testing skipping duplicates xxx@blabla+ <p>[email protected]</p>+ [email protected] testing skipping duplicates xxx@blabla<p>carl@example.com</p><a href="mailto:[email protected]"></a>- <a href="mailto:david@example.com"></a>-- <a href="[email protected]"></a>+ <a href="mailto:david@example.com"></a>++ <a href="[email protected]"></a><img src="http://somewhere.com/ [email protected] " /><a href="http://somewhere.com/ [email protected] "></a>-- +420775222222++ +420775222222+4207751111111+4207751111111 test duplicate1 2 3 4 5 6 7 8 9 0<a href="skip.this.one: +42099999999"></a><a href="tel://+42077533333"></a>-+https://www.linkedin.com/in/bobnewmanhttps://www.linkedin.com/in/alicenewman/https://www.linkedin.com/in/alicenewman/ duplicate@@ -363,26 +366,26 @@ describe('utils.social', () => {https://www.linkedin.com/in/first-last-123456a<a href="https://www.linkedin.com/in/jancurn">Profile</a><a href="https://www.linkedin.com/in/carl-newman-5555555a/detail/recent-activity/">Sub-link</a>-+https://www.instagram.com/old_prague/https://www.instagram.com/old_prague/ duplicate- instagram.com/old_prague- https://www.instagram.com/newyorkarea/+ instagram.com/old_prague+ https://www.instagram.com/newyorkarea/<a href="https://www.instagram.com/york">link</a><a href="https://www.instagram.com/york2/something">sub-link</a>-+<a href="twitter.com/betasomething">link</a>https://www.twitter.com/apify- https://www.twitter.com/apify duplicate+ https://www.twitter.com/apify duplicate<a href="twitter.com/cblabla/sub-dir/">link</a>-- <a href="facebook.com/carl.username123/sub-dir/">link</a>++ <a href="facebook.com/carl.username123/sub-dir/">link</a>https://www.facebook.com/bob.username123/https://www.facebook.com/bob.username123/ duplicatehttp://www.facebook.com/alice.username123<a href="https://www.facebook.com/profile.php?id=1155802082&xxx=5">link x</a><a href="fb.com/dada5678?query=1">link</a>-+</body></html>`)).to.eql({@@ -426,7 +429,7 @@ describe('utils.social', () => {social.parseHandlesFromHtml(`<html><head>- <title>Bla</title>+ <title>Bla</title></head><body>Body content@@ -515,7 +518,7 @@ describe('utils.social', () => {expect(social.LINKEDIN_REGEX_GLOBAL.test('https://www.linkedin.com/in/bobnewman')).to.eql(true);expect(`- https://www.linkedin.com/in/bobnewman+ https://www.linkedin.com/in/bobnewman"http://ie.linkedin.com/in/alicenewman"https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamelinkedin.com/in/carlnewman@@ -584,7 +587,7 @@ describe('utils.social', () => {expect(social.INSTAGRAM_REGEX_GLOBAL.test('https://www.instagram.com/old_prague')).to.eql(true);expect(`https://www.instagram.com/old_prague- https://www.instagram.com/someverylongusernamethatisnotgood+ https://www.instagram.com/someverylongusernamethatisnotgood"instagram.com/old_brno"http://instagr.am/old_plzen`.match(social.INSTAGRAM_REGEX_GLOBAL))