Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added getPublicUrl function

Aaron Jacksonbb12e97f90af
diff --git a/package.json b/package.json
index 4786b398d..ca17a197e 100644
--- a/package.json
+++ b/package.json
@@ -79,7 +79,6 @@
"eslint-config-airbnb-base": "^13.1.0",
"eslint-plugin-import": "^2.14.0",
"eslint-plugin-jsx-a11y": "^6.1.2",
- "eslint-plugin-mocha": "^5.2.0",
"eslint-plugin-promise": "^4.0.1",
"eslint-plugin-react": "^7.11.1",
"jsdoc": "^3.5.5",
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 2f78ddf1a..ce0563cf7 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -25,6 +25,7 @@ const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.
+const publicUrl = 'https://api.apify.com/v2/key-value-stores' // Will be replaced with env var in future.
/**
* Helper function to validate params of *.getValue().
@@ -291,9 +292,8 @@ export class KeyValueStore {
}
getPublicUrl(fileName) {
- ensureTokenOrLocalStorageEnvExists('key value store');
- const publicUrl = `https://api.apify.com/v2/key-value-stores/${this.storeId}/records/${fileName}`;
- return publicUrl;
+ ensureTokenOrLocalStorageEnvExists('key value store');
+ return publicUrl + `${this.storeId}/records/${fileName}`;;
}
}
@@ -441,7 +441,8 @@ export class KeyValueStoreLocal {
getPublicUrl(fileName) {
ensureTokenOrLocalStorageEnvExists('key value store');
const publicLocalUrl = `file://${this.localStoragePath}/${fileName}`;
- return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : publicUrl;
+ const defaultUrl = `file://${this.localStoragePath}/default`
+ return process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !this.forceCloud ? publicLocalUrl : defaultUrl;
}
}