Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added getPublicUrl function
Aaron Jacksonbb12e97f90af
diff --git a/package.json b/package.jsonindex 4786b398d..ca17a197e 100644--- a/package.json+++ b/package.json@@ -79,7 +79,6 @@"eslint-config-airbnb-base": "^13.1.0","eslint-plugin-import": "^2.14.0","eslint-plugin-jsx-a11y": "^6.1.2",- "eslint-plugin-mocha": "^5.2.0","eslint-plugin-promise": "^4.0.1","eslint-plugin-react": "^7.11.1","jsdoc": "^3.5.5",diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 2f78ddf1a..ce0563cf7 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -25,6 +25,7 @@ const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);const { keyValueStores } = apifyClient;const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.+const publicUrl = 'https://api.apify.com/v2/key-value-stores' // Will be replaced with env var in future./*** Helper function to validate params of *.getValue().@@ -291,9 +292,8 @@ export class KeyValueStore {}getPublicUrl(fileName) {- ensureTokenOrLocalStorageEnvExists('key value store');- const publicUrl = `https://api.apify.com/v2/key-value-stores/${this.storeId}/records/${fileName}`;- return publicUrl;+ ensureTokenOrLocalStorageEnvExists('key value store');+ return publicUrl + `${this.storeId}/records/${fileName}`;;}}@@ -441,7 +441,8 @@ export class KeyValueStoreLocal {getPublicUrl(fileName) {ensureTokenOrLocalStorageEnvExists('key value store');const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : publicUrl;+ const defaultUrl = `file://${this.localStoragePath}/default`+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : defaultUrl;}}