Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add getPublicUrl function to key-value-store

aarontjdev0fd053489a74
diff --git a/src/key_value_store.js b/src/key_value_store.js
index e9e9d3f83..051e63337 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -438,11 +438,11 @@ export class KeyValueStoreLocal {
* Retrieves the full public url of the designated file.
* @param {String} fileName
*/
- getPublicUrl(fileName) {
+ getPublicUrl(fileName, store_id) {
+ store_id = store_id ? store_id : 'default';
ensureTokenOrLocalStorageEnvExists('key value store');
- const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;
- const defaultUrl = `file://${this.localStoragePath}/default`;
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : defaultUrl;
+ const publicLocalUrl = `file:/${this.localStoragePath}/${store_id}/${fileName}`;
+ return publicLocalUrl;
}
}
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 0bd0edf2d..6551ffee3 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -8,6 +8,7 @@ import { KeyValueStoreLocal, KeyValueStore, maybeStringify, getFileNameRegexp, L
import * as utils from '../build/utils';
import * as Apify from '../build/index';
import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNonEmpty } from './_helper';
+import { doesNotReject } from 'assert';
const { apifyClient } = utils;
@@ -365,4 +366,23 @@ describe('KeyValueStore', () => {
mock.restore();
});
});
+
+ describe('getPublicUrl', () => {
+ it('should return the local url of a file', () => {
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
+
+ expect(store.getPublicUrl('file','my-store-id')).to.equal(`file:/${store.localStoragePath}/my-store-id/file`);
+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
+ });
+
+ it('should return the url of a file in apify cloud', async () => {
+ process.env[ENV_VARS.TOKEN] = 'xxx';
+ const publicUrl = 'https://api.apify.com/v2/key-value-stores';
+ const store = new KeyValueStore('my-store-id');
+
+ expect(store.getPublicUrl('file')).to.equal(`${publicUrl}/my-store-id/records/file`);
+ delete process.env[ENV_VARS.TOKEN];
+ })
+ });
});