Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added getPublicUrl function to key_value_store
Aaron Jacksonfa875f269c0f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 692916d16..93c36979d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+0.9.14 / 2018-11-26+===================+- Added `_getPublicUrl` function to `KeyValueStoreLocal`+0.9.13 / 2018-11-26===================- Updated NPM packages to fix a vulnerability reported at https://github.com/dominictarr/event-stream/issues/116diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 72658dc95..3fcf7b663 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -427,6 +427,20 @@ export class KeyValueStoreLocal {_getPath(fileName) {return path.resolve(this.localStoragePath, fileName);}++ /**+ * Retrieves the full public url of the designated file.+ * @param {String} fileName+ */+ _getPublicUrl( fileName ) {+ ensureTokenOrLocalStorageEnvExists('key value store');+ var storeId = process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];++ const publicUrl = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${fileName}`;+ const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;++ return process.env[_consts.ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : publicUrl;+ }}/**