Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
add getPublicUrl function to key-value-store
aarontjdev0fd053489a74
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex e9e9d3f83..051e63337 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -438,11 +438,11 @@ export class KeyValueStoreLocal {* Retrieves the full public url of the designated file.* @param {String} fileName*/- getPublicUrl(fileName) {+ getPublicUrl(fileName, store_id) {+ store_id = store_id ? store_id : 'default';ensureTokenOrLocalStorageEnvExists('key value store');- const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;- const defaultUrl = `file://${this.localStoragePath}/default`;- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : defaultUrl;+ const publicLocalUrl = `file:/${this.localStoragePath}/${store_id}/${fileName}`;+ return publicLocalUrl;}}diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 0bd0edf2d..6551ffee3 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -8,6 +8,7 @@ import { KeyValueStoreLocal, KeyValueStore, maybeStringify, getFileNameRegexp, Limport * as utils from '../build/utils';import * as Apify from '../build/index';import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir, expectDirEmpty, expectDirNonEmpty } from './_helper';+import { doesNotReject } from 'assert';const { apifyClient } = utils;@@ -365,4 +366,23 @@ describe('KeyValueStore', () => {mock.restore();});});++ describe('getPublicUrl', () => {+ it('should return the local url of a file', () => {+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);++ expect(store.getPublicUrl('file','my-store-id')).to.equal(`file:/${store.localStoragePath}/my-store-id/file`);+ delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];+ });++ it('should return the url of a file in apify cloud', async () => {+ process.env[ENV_VARS.TOKEN] = 'xxx';+ const publicUrl = 'https://api.apify.com/v2/key-value-stores';+ const store = new KeyValueStore('my-store-id');++ expect(store.getPublicUrl('file')).to.equal(`${publicUrl}/my-store-id/records/file`);+ delete process.env[ENV_VARS.TOKEN];+ })+ });});