Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added getPublicUrl function to key_value_store

Aaron Jacksonfa875f269c0f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 692916d16..93c36979d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+0.9.14 / 2018-11-26
+===================
+- Added `_getPublicUrl` function to `KeyValueStoreLocal`
+
0.9.13 / 2018-11-26
===================
- Updated NPM packages to fix a vulnerability reported at https://github.com/dominictarr/event-stream/issues/116
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 72658dc95..3fcf7b663 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -427,6 +427,20 @@ export class KeyValueStoreLocal {
_getPath(fileName) {
return path.resolve(this.localStoragePath, fileName);
}
+
+ /**
+ * Retrieves the full public url of the designated file.
+ * @param {String} fileName
+ */
+ _getPublicUrl( fileName ) {
+ ensureTokenOrLocalStorageEnvExists('key value store');
+ var storeId = process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
+
+ const publicUrl = `https://api.apify.com/v2/key-value-stores/${storeId}/records/${fileName}`;
+ const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;
+
+ return process.env[_consts.ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : publicUrl;
+ }
}
/**