Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Make changes

Ondra Urbanf77e34feee1c
diff --git a/src/constants.js b/src/constants.js
index b9a33d343..66d09f67f 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -120,3 +120,9 @@ export const USER_AGENT_LIST = [
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 OPR/56.0.3051.99',
];
+
+/**
+ * Base URL of Apify's API endpoints.
+ * @type {string}
+ */
+export const APIFY_API_BASE_URL = 'https://api.apify.com/v2';
diff --git a/src/key_value_store.js b/src/key_value_store.js
index eab17e175..830b31257 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -11,6 +11,7 @@ import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
import {
addCharsetToContentType, apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists,
} from './utils';
+import { APIFY_API_BASE_URL } from './constants';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
@@ -26,7 +27,6 @@ const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.
-const publicUrl = 'https://api.apify.com/v2/key-value-stores'; // Will be replaced with env var in future.
/**
* Helper function to validate params of *.getValue().
@@ -293,10 +293,15 @@ export class KeyValueStore {
});
}
-
- getPublicUrl(fileName) {
- ensureTokenOrLocalStorageEnvExists('key value store');
- return `${publicUrl}/${this.storeId}/records/${fileName}`;
+ /**
+ * Returns a URL for the given key that may be used to publicly
+ * access the value in the remote key value store.
+ *
+ * @param {string} key
+ * @return {string}
+ */
+ getPublicUrl(key) {
+ return `${APIFY_API_BASE_URL}/key-value-stores/${this.storeId}/records/${key}`;
}
/**
@@ -521,16 +526,17 @@ export class KeyValueStoreLocal {
}
/**
- * Retrieves the full public url of the designated file.
- * @param {String} fileName
+ * Returns a file:// URL for the given fileName that may be used to
+ * access the value on the local drive.
+ *
+ * Unlike in the remote store where key is sufficient, a full fileName
+ * must be provided here including the extension for the URL to be valid.
+ *
+ * @param {string} fileName
+ * @return {string}
*/
- getPublicUrl(fileName, storeId) {
- if (!storeId) {
- storeId = 'default';
- }
- ensureTokenOrLocalStorageEnvExists('key value store');
- const publicLocalUrl = `file:/${this.localStoragePath}/${storeId}/${fileName}`;
- return publicLocalUrl;
+ getPublicUrl(fileName) {
+ return `file://${this._getPath(fileName)}`;
}
}
diff --git a/test/key_value_store.js b/test/key_value_store.js
index bc116b292..205b7ee28 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -373,7 +373,7 @@ describe('KeyValueStore', () => {
it('should return the local url of a file', () => {
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
const store = new KeyValueStoreLocal('my-store-id', LOCAL_STORAGE_DIR);
- expect(store.getPublicUrl('file', 'my-store-id')).to.equal(`file:/${store.localStoragePath}/my-store-id/file`);
+ expect(store.getPublicUrl('file.txt')).to.equal(`file://${store.localStoragePath}/file.txt`);
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
});