Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: automatically convert google spreadsheet URLs to CSV exports (#1255)
Martin Adámek1c5ed7665da1
diff --git a/package.json b/package.jsonindex 35140a97e..d8fab33ac 100644--- a/package.json+++ b/package.json@@ -126,7 +126,7 @@"sinon": "^12.0.0","sinon-stub-promise": "^4.0.0","ts-jest": "^27.0.5",- "typescript": "4.5.4",+ "typescript": "4.4.4","ua-parser-js": "^1.0.0","why-is-node-running": "^2.2.0"}diff --git a/src/request_list.js b/src/request_list.jsindex 6f3505354..f83ad70c3 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -652,47 +652,24 @@ export class RequestList {async _fetchRequestsFromUrl(source) {const { requestsFromUrl, regex, ...sharedOpts } = source;const { downloadListOfUrls } = publicUtils;- const fixedRequestsFromUrl = this._ensureCorrectRemoteRequestListUrl(requestsFromUrl);-- if (requestsFromUrl !== fixedRequestsFromUrl) {- this.log.warning('Detected wrong request list url format, fixing automatically.', { old: requestsFromUrl, new: fixedRequestsFromUrl });- }// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await downloadListOfUrls({ url: fixedRequestsFromUrl, urlRegExp: regex });+ urlsArr = await downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex });} catch (err) {- throw new Error(`Cannot fetch a request list from ${fixedRequestsFromUrl}: ${err}`);+ throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);}// Skip if resource contained no URLs.if (!urlsArr.length) {- this.log.warning('list fetched, but it is empty.', { requestsFromUrl: fixedRequestsFromUrl, regex });+ this.log.warning('list fetched, but it is empty.', { requestsFromUrl, regex });return [];}return urlsArr.map((url) => ({ url, ...sharedOpts }));}- /**- * Tries to detect wrong urls and fix them. Currently detects only sharing url instead of csv download one.- * @param {string} url- * @return {string}- * @ignore- * @internal- * @private- */- _ensureCorrectRemoteRequestListUrl(url) {- const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/edit/);-- if (match) {- return `${match[1]}/gviz/tq?tqx=out:csv`;- }-- return url;- }-/*** Adds given request.* If the `source` parameter is a string or plain object and not an instancediff --git a/src/utils.js b/src/utils.jsindex 8e6074c13..dd96fe1cc 100644--- a/src/utils.js+++ b/src/utils.js@@ -414,7 +414,16 @@ const downloadListOfUrls = async (options) => {urlRegExp: ow.optional.regExp,}));const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX } = options;- const { body: string } = await requestAsBrowser({ url, encoding });++ // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.+ const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);+ let fixedUrl = url;++ if (match) {+ fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;+ }++ const { body: string } = await requestAsBrowser({ url: fixedUrl, encoding });return extractUrls({ string, urlRegExp });};diff --git a/test/request_list.test.js b/test/request_list.test.jsindex a58bdd828..5483cb75a 100644--- a/test/request_list.test.js+++ b/test/request_list.test.js@@ -198,16 +198,23 @@ describe('Apify.RequestList', () => {'https://google.com','https://wired.com',];- const wrongUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing';+ const wrongUrls = [+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/123123132',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/?q=blablabla',+ 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit#gid=0',+ ];const correctUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv';mock.expects('requestAsBrowser')- .once()+ .exactly(wrongUrls.length).withArgs({ url: correctUrl, encoding: 'utf8' }).resolves({ body: JSON.stringify(list) });const requestList = new Apify.RequestList({- sources: [{ requestsFromUrl: wrongUrl }],+ sources: wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl })),});await requestList.initialize();