Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: automatically convert gdoc share urls to csv download ones in req. list (#1174)
Martin Adámekad79fee490ef
diff --git a/src/request_list.js b/src/request_list.jsindex 9e7b0c1e6..a9d5fcd7c 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -650,25 +650,47 @@ export class RequestList {* @internal*/async _fetchRequestsFromUrl(source) {- const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');- const { requestsFromUrl, regex } = source;+ const { requestsFromUrl, regex, ...sharedOpts } = source;const { downloadListOfUrls } = publicUtils;+ const fixedRequestsFromUrl = this._ensureCorrectRemoteRequestListUrl(requestsFromUrl);++ if (requestsFromUrl !== fixedRequestsFromUrl) {+ this.log.warning('Detected wrong request list url format, fixing automatically.', { old: requestsFromUrl, new: fixedRequestsFromUrl });+ }// Download remote resource and parse URLs.let urlsArr;try {- urlsArr = await downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex });+ urlsArr = await downloadListOfUrls({ url: fixedRequestsFromUrl, urlRegExp: regex });} catch (err) {- throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);+ throw new Error(`Cannot fetch a request list from ${fixedRequestsFromUrl}: ${err}`);}// Skip if resource contained no URLs.if (!urlsArr.length) {- this.log.warning('list fetched, but it is empty.', { requestsFromUrl, regex });+ this.log.warning('list fetched, but it is empty.', { requestsFromUrl: fixedRequestsFromUrl, regex });return [];}- return urlsArr.map((url) => _.extend({ url }, sharedOpts));+ return urlsArr.map((url) => ({ url, ...sharedOpts }));+ }++ /**+ * Tries to detect wrong urls and fix them. Currently detects only sharing url instead of csv download one.+ * @param {string} url+ * @return {string}+ * @ignore+ * @internal+ * @private+ */+ _ensureCorrectRemoteRequestListUrl(url) {+ const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/edit/);++ if (match) {+ return `${match[1]}/gviz/tq?tqx=out:csv`;+ }++ return url;}/**diff --git a/test/request_list.test.js b/test/request_list.test.jsindex 535d6318c..a58bdd828 100644--- a/test/request_list.test.js+++ b/test/request_list.test.js@@ -191,6 +191,35 @@ describe('Apify.RequestList', () => {mock.restore();});+ test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => {+ const mock = sinon.mock(requestUtils);+ const list = [+ 'https://example.com',+ 'https://google.com',+ 'https://wired.com',+ ];+ const wrongUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing';+ const correctUrl = 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv';++ mock.expects('requestAsBrowser')+ .once()+ .withArgs({ url: correctUrl, encoding: 'utf8' })+ .resolves({ body: JSON.stringify(list) });++ const requestList = new Apify.RequestList({+ sources: [{ requestsFromUrl: wrongUrl }],+ });++ await requestList.initialize();++ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] });+ expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] });++ mock.verify();+ mock.restore();+ });+test('should handle requestsFromUrl with no URLs', async () => {const mock = sinon.mock(utils.publicUtils);mock.expects('downloadListOfUrls')