Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: use `@apify/tsconfig` with enabled `esModulesInterop` (#1182)
Martin Adámeka8feb28de376
diff --git a/package.json b/package.jsonindex 14ecf7612..5d1195333 100644--- a/package.json+++ b/package.json@@ -6,7 +6,7 @@"node": ">=15.10.0"},"main": "build/index.js",- "types": "types/index.d.ts",+ "types": "build/index.d.ts","keywords": ["apify","headless",@@ -28,16 +28,14 @@"license": "Apache-2.0","repository": {"type": "git",- "url": "git+https://github.com/apifytech/apify-js"+ "url": "git+https://github.com/apify/apify-js"},"bugs": {- "url": "https://github.com/apifytech/apify-js/issues"+ "url": "https://github.com/apify/apify-js/issues"},"homepage": "https://sdk.apify.com/","files": [- "build",- "types",- "types-apify"+ "build"],"scripts": {"build": "npm run clean && tsc -p tsconfig.json && node ./tools/typescript_fixes.js",@@ -46,7 +44,7 @@"test": "npm run build && jest","prepare": "npm run build","prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",- "clean": "rimraf ./build ./types",+ "clean": "rimraf ./build","lint": "eslint ./src ./test","lint:fix": "eslint ./src ./test --ext .js,.jsx --fix"},@@ -87,6 +85,7 @@},"devDependencies": {"@apify/eslint-config": "^0.1.4",+ "@apify/tsconfig": "^0.1.0","@babel/cli": "^7.14.8","@babel/core": "^7.14.8","@babel/eslint-parser": "^7.14.7",@@ -99,6 +98,7 @@"@types/htmlparser2": "^3.10.3","@types/jest": "^27.0.1","@types/node": "^15.14.2",+ "@types/ps-tree": "^1.1.1","@types/request-promise-native": "^1.0.18","@types/rimraf": "^3.0.1","@types/semver": "^7.3.7",diff --git a/src/actor.js b/src/actor.jsindex b5c320239..77a28fc3e 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,6 +1,6 @@import ow from 'ow';-import * as path from 'path';-import * as _ from 'underscore';+import path from 'path';+import _ from 'underscore';import { ENV_VARS, INTEGER_ENV_VARS, ACT_JOB_STATUSES } from '@apify/consts';import log from './utils_log';import { EXIT_CODES } from './constants';@@ -13,8 +13,8 @@ import {isAtHome,logSystemInfo,printOutdatedSdkWarning,+ newClient,} from './utils';-import * as utils from './utils';import { maybeStringify } from './storages/key_value_store';// eslint-disable-next-line import/named,no-unused-vars,import/first@@ -334,7 +334,7 @@ export const call = async (actId, input, options = {}) => {input = maybeStringify(input, callActorOpts);}- const client = token ? utils.newClient({ token }) : apifyClient;+ const client = token ? newClient({ token }) : apifyClient;let run;try {@@ -458,7 +458,7 @@ export const callTask = async (taskId, input, options = {}) => {callTaskOpts.memory = memoryMbytes;callTaskOpts.timeout = timeoutSecs;- const client = token ? utils.newClient({ token }) : apifyClient;+ const client = token ? newClient({ token }) : apifyClient;// Start task and wait for run to finish if waitSecs is providedlet run;try {@@ -551,7 +551,7 @@ export const metamorph = async (targetActorId, input, options = {}) => {input = maybeStringify(input, metamorphOpts);}- await utils.apifyClient.run(runId, actorId).metamorph(targetActorId, input, metamorphOpts);+ await apifyClient.run(runId, actorId).metamorph(targetActorId, input, metamorphOpts);// Wait some time for container to be stopped.// NOTE: option.customAfterSleepMillis is used in testsdiff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 151ce0d4e..15362e7c5 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -1,4 +1,4 @@-import * as os from 'os';+import os from 'os';import ow from 'ow';import { betterSetInterval, betterClearInterval } from '@apify/utilities';import { ACTOR_EVENT_NAMES, ENV_VARS } from '@apify/consts';diff --git a/src/configuration.js b/src/configuration.jsindex e41b2f1b7..055b7f283 100644--- a/src/configuration.js+++ b/src/configuration.js@@ -1,7 +1,7 @@import { ENV_VARS, LOCAL_ENV_VARS } from '@apify/consts';import { join } from 'path';import { ApifyStorageLocal } from '@apify/storage-local';-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';import log from './utils_log';/**diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 243792589..cbf67a1a0 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -1,7 +1,7 @@import { ACTOR_EVENT_NAMES } from '@apify/consts';import { cryptoRandomObjectId } from '@apify/utilities';import ow, { ArgumentError } from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-line import/no-duplicatesimport events from '../events';import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicatesdiff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 3400f62e0..6798820e9 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -1,16 +1,16 @@/* eslint-disable class-methods-use-this */import { readStreamToString, concatStreamToBuffer } from '@apify/utilities';-import * as cheerio from 'cheerio';-import * as contentTypeParser from 'content-type';-import * as htmlparser from 'htmlparser2';+import cheerio from 'cheerio';+import contentTypeParser from 'content-type';+import { DomHandler } from 'htmlparser2';import { WritableStream } from 'htmlparser2/lib/WritableStream';-import * as iconv from 'iconv-lite';+import iconv from 'iconv-lite';import ow from 'ow';-import * as util from 'util';+import util from 'util';import { TimeoutError } from 'got-scraping';import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';-import * as utilsRequest from '../utils_request'; // eslint-disable-line import/no-duplicates+import { requestAsBrowser } from '../utils_request';import { BasicCrawler } from './basic_crawler'; // eslint-disable-line import/no-duplicatesimport CrawlerExtension from './crawler_extension';@@ -655,7 +655,7 @@ class CheerioCrawler extends BasicCrawler {let responseWithStream;try {- responseWithStream = await utilsRequest.requestAsBrowser(opts);+ responseWithStream = await requestAsBrowser(opts);} catch (e) {if (e instanceof TimeoutError) {this._handleRequestTimeout(session);@@ -796,7 +796,7 @@ class CheerioCrawler extends BasicCrawler {*/async _parseHtmlToDom(response) {return new Promise((resolve, reject) => {- const domHandler = new htmlparser.DomHandler((err, dom) => {+ const domHandler = new DomHandler((err, dom) => {if (err) reject(err);else resolve(dom);});diff --git a/src/enqueue_links/shared.js b/src/enqueue_links/shared.jsindex d28894919..8f3f712fe 100644--- a/src/enqueue_links/shared.js+++ b/src/enqueue_links/shared.js@@ -1,5 +1,5 @@import { URL } from 'url';-import * as _ from 'underscore';+import _ from 'underscore';import PseudoUrl from '../pseudo_url';import Request from '../request'; // eslint-disable-line import/no-duplicatesdiff --git a/src/events.js b/src/events.jsindex 162fec8f0..ad3e04f50 100644--- a/src/events.js+++ b/src/events.js@@ -1,5 +1,5 @@import { EventEmitter } from 'events';-import * as WebSocket from 'ws';+import WebSocket from 'ws';import { ENV_VARS, ACTOR_EVENT_NAMES } from '@apify/consts';import { ACTOR_EVENT_NAMES_EX } from './constants';import defaultLog from './utils_log';diff --git a/src/playwright_utils.js b/src/playwright_utils.jsindex cbc737793..87cc4ab83 100644--- a/src/playwright_utils.js+++ b/src/playwright_utils.js@@ -1,5 +1,5 @@import ow from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import { Page, Response } from 'playwright'; // eslint-disable-line no-unused-varsimport log from './utils_log';import { validators } from './validators';diff --git a/src/pseudo_url.js b/src/pseudo_url.jsindex 762030018..48f646930 100644--- a/src/pseudo_url.js+++ b/src/pseudo_url.js@@ -1,5 +1,5 @@import ow from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import log from './utils_log';import Request, { RequestOptions } from './request'; // eslint-disable-line import/named,no-unused-varsdiff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.jsindex 785708b5f..f53deeb10 100644--- a/src/puppeteer_request_interception.js+++ b/src/puppeteer_request_interception.js@@ -1,6 +1,6 @@import { EventEmitter } from 'events';import ow from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import { Request as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable-line no-unused-vars// We use weak maps here so that the content gets discarted after page gets closed.diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex e49ec8b80..322eaa2d9 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -1,8 +1,8 @@-import * as fs from 'fs';+import fs from 'fs';import ow from 'ow';-import * as vm from 'vm';-import * as util from 'util';-import * as _ from 'underscore';+import vm from 'vm';+import util from 'util';+import _ from 'underscore';import { LruCache } from '@apify/datastructures';import { Page, Response, DirectNavigationOptions } from 'puppeteer'; // eslint-disable-line no-unused-varsimport log from './utils_log';diff --git a/src/request.js b/src/request.jsindex 4fc5f684c..e2fda3979 100644--- a/src/request.js+++ b/src/request.js@@ -1,7 +1,7 @@import { normalizeUrl } from '@apify/utilities';-import * as crypto from 'crypto';+import crypto from 'crypto';import ow, { ArgumentError } from 'ow';-import * as util from 'util';+import util from 'util';import defaultLog from './utils_log';// new properties on the Request object breaks serializationdiff --git a/src/request_list.js b/src/request_list.jsindex a9d5fcd7c..22f56968b 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -1,5 +1,5 @@import ow, { ArgumentError } from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import { ACTOR_EVENT_NAMES_EX } from './constants';import Request from './request'; // eslint-disable-line import/no-duplicatesimport events from './events';diff --git a/src/serialization.js b/src/serialization.jsindex 2a8a78fb3..7514c4428 100644--- a/src/serialization.js+++ b/src/serialization.js@@ -1,8 +1,8 @@import ow from 'ow';-import * as stream from 'stream'; // eslint-disable-line import/no-duplicates-import * as StreamArray from 'stream-json/streamers/StreamArray';-import * as util from 'util';-import * as zlib from 'zlib';+import stream from 'stream'; // eslint-disable-line import/no-duplicates+import StreamArray from 'stream-json/streamers/StreamArray';+import util from 'util';+import zlib from 'zlib';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */diff --git a/src/stealth/stealth.js b/src/stealth/stealth.jsindex 6e04c9518..44d7171d3 100644--- a/src/stealth/stealth.js+++ b/src/stealth/stealth.js@@ -1,4 +1,4 @@-import * as _ from 'underscore';+import _ from 'underscore';import { Page, Browser } from 'puppeteer'; // eslint-disable-line no-unused-varsimport { cryptoRandomObjectId } from '@apify/utilities';import globalLog from '../utils_log';diff --git a/src/storages/dataset.js b/src/storages/dataset.jsindex 6ae24b73a..0ad1484be 100644--- a/src/storages/dataset.js+++ b/src/storages/dataset.js@@ -1,12 +1,12 @@import ow from 'ow';-import * as _ from 'underscore';+import _ from 'underscore';import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';import { StorageManager } from './storage_manager';import log from '../utils_log';/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */// @ts-ignore-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';// @ts-ignoreimport { ApifyStorageLocal } from '@apify/storage-local';import { Configuration } from '../configuration';diff --git a/src/storages/key_value_store.js b/src/storages/key_value_store.jsindex 206e8eadc..5e146d310 100644--- a/src/storages/key_value_store.js+++ b/src/storages/key_value_store.js@@ -6,7 +6,7 @@ import log from '../utils_log';/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */// @ts-ignore-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';// @ts-ignoreimport { ApifyStorageLocal } from '@apify/storage-local';import { Configuration } from '../configuration';diff --git a/src/storages/request_queue.js b/src/storages/request_queue.jsindex 872a814a1..6e6993858 100644--- a/src/storages/request_queue.js+++ b/src/storages/request_queue.js@@ -1,4 +1,4 @@-import * as crypto from 'crypto';+import crypto from 'crypto';import { ListDictionary, LruCache } from '@apify/datastructures';import { REQUEST_QUEUE_HEAD_MAX_LIMIT } from '@apify/consts';import { cryptoRandomObjectId } from '@apify/utilities';@@ -10,7 +10,7 @@ import log from '../utils_log';/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */import Request, { RequestOptions } from '../request'; // eslint-disable-line import/named,no-unused-vars// @ts-ignore-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';// @ts-ignoreimport { ApifyStorageLocal } from '@apify/storage-local';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */diff --git a/src/storages/storage_manager.js b/src/storages/storage_manager.jsindex d53e0d5a4..2bc5d4e01 100644--- a/src/storages/storage_manager.js+++ b/src/storages/storage_manager.js@@ -2,7 +2,7 @@ import { ENV_VARS } from '@apify/consts';/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */// @ts-ignore-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';// @ts-ignoreimport { ApifyStorageLocal } from '@apify/storage-local';import cacheContainer from '../cache_container';diff --git a/src/utils.js b/src/utils.jsindex 358c6f16a..c8cff436c 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,21 +1,21 @@-import * as psTree from '@apify/ps-tree';+import psTree from '@apify/ps-tree';// eslint-disable-next-line import/no-extraneous-dependenciesimport { execSync } from 'child_process';-import * as ApifyClient from 'apify-client';+import ApifyClient from 'apify-client';import { version as apifyClientVersion } from 'apify-client/package.json';import { ACT_JOB_TERMINAL_STATUSES, ENV_VARS } from '@apify/consts';-import * as cheerio from 'cheerio';-import * as contentTypeParser from 'content-type';-import * as fs from 'fs';-import * as mime from 'mime-types';-import * as os from 'os';+import cheerio from 'cheerio';+import contentTypeParser from 'content-type';+import fs from 'fs';+import mime from 'mime-types';+import os from 'os';// eslint-disable-next-line import/no-extraneous-dependenciesimport ow from 'ow';-import * as path from 'path';-import * as semver from 'semver';-import * as _ from 'underscore';+import path from 'path';+import semver from 'semver';+import _ from 'underscore';import { URL } from 'url';-import * as util from 'util';+import util from 'util';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */@@ -23,7 +23,7 @@ import { IncomingMessage } from 'http';import { Response as PuppeteerResponse } from 'puppeteer';import { version as apifyVersion } from '../package.json';import log from './utils_log';-import * as requestUtils from './utils_request';+import { requestAsBrowser } from './utils_request';import Request, { RequestOptions } from './request';import { ActorRun } from './typedefs';@@ -400,9 +400,6 @@ const downloadListOfUrls = async (options) => {urlRegExp: ow.optional.regExp,}));const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX } = options;-- const { requestAsBrowser } = requestUtils;-const { body: string } = await requestAsBrowser({ url, encoding });return extractUrls({ string, urlRegExp });};diff --git a/src/utils_social.js b/src/utils_social.jsindex e1cdd0ac4..d0002e7fd 100644--- a/src/utils_social.js+++ b/src/utils_social.js@@ -1,6 +1,6 @@/* eslint-disable no-continue */-import * as _ from 'underscore';-import * as cheerio from 'cheerio';+import _ from 'underscore';+import cheerio from 'cheerio';import log from './utils_log';import { publicUtils } from './utils';diff --git a/test/autoscaling/snapshotter.test.js b/test/autoscaling/snapshotter.test.jsindex ce1df6529..55b4a8e62 100644--- a/test/autoscaling/snapshotter.test.js+++ b/test/autoscaling/snapshotter.test.js@@ -4,7 +4,7 @@ import os from 'os';import sinon from 'sinon';import { ACTOR_EVENT_NAMES, ENV_VARS } from '@apify/consts';import log from '../../build/utils_log';-import * as Apify from '../../build/index';+import Apify from '../../build/index';import events from '../../build/events';import Snapshotter from '../../build/autoscaling/snapshotter';import * as utils from '../../build/utils';diff --git a/test/crawlers/basic_crawler.test.js b/test/crawlers/basic_crawler.test.jsindex ff7ec3b44..cdda059eb 100644--- a/test/crawlers/basic_crawler.test.js+++ b/test/crawlers/basic_crawler.test.js@@ -2,7 +2,7 @@ import _ from 'underscore';import sinon from 'sinon';import { ACTOR_EVENT_NAMES } from '@apify/consts';import log from '../../build/utils_log';-import * as Apify from '../../build';+import Apify from '../../build';import * as keyValueStore from '../../build/storages/key_value_store';import { RequestQueue } from '../../build/storages/request_queue';import events from '../../build/events';diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.jsindex aac942477..0e21ac275 100644--- a/test/crawlers/browser_crawler.test.js+++ b/test/crawlers/browser_crawler.test.js@@ -6,7 +6,7 @@ import sinon from 'sinon';import { BrowserPool, PuppeteerPlugin } from 'browser-pool';import puppeteer from 'puppeteer';import log from '../../build/utils_log';-import * as Apify from '../../build';+import Apify from '../../build';import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 40e09900f..71f61ded0 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -7,7 +7,7 @@ import express from 'express';import bodyParser from 'body-parser';import sinon from 'sinon';import { Readable } from 'stream';-import * as iconv from 'iconv-lite';+import iconv from 'iconv-lite';import log from '../../build/utils_log';import Apify from '../../build';import { sleep } from '../../build/utils';diff --git a/test/crawlers/playwright_crawler.test.js b/test/crawlers/playwright_crawler.test.jsindex c38e6ad1f..e12a09919 100644--- a/test/crawlers/playwright_crawler.test.js+++ b/test/crawlers/playwright_crawler.test.js@@ -1,7 +1,7 @@import { ENV_VARS } from '@apify/consts';import playwright from 'playwright';import log from '../../build/utils_log';-import * as Apify from '../../build';+import Apify from '../../build';import LocalStorageDirEmulator from '../local_storage_dir_emulator';describe('PlaywrightCrawler', () => {diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex 916fd687b..71342072e 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -1,7 +1,7 @@import { ENV_VARS } from '@apify/consts';import sinon from 'sinon';import log from '../../build/utils_log';-import * as Apify from '../../build';+import Apify from '../../build';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utils from '../../build/utils';diff --git a/test/enqueue_links/shared.test.js b/test/enqueue_links/shared.test.jsindex 948508f9e..9697bc937 100644--- a/test/enqueue_links/shared.test.js+++ b/test/enqueue_links/shared.test.js@@ -1,5 +1,5 @@import PseudoUrl from '../../build/pseudo_url';-import * as shared from '../../build/enqueue_links/shared';+import { constructPseudoUrlInstances, createRequestOptions, createRequests, addRequestsToQueueInBatches } from '../../build/enqueue_links/shared';describe('Enqueue links shared functions', () => {describe('constructPseudoUrlInstances()', () => {@@ -10,7 +10,7 @@ describe('Enqueue links shared functions', () => {'http[s?]://example.com/[.*]',{ purl: 'http[s?]://example.com[.*]', userData: { foo: 'bar' } },];- const pseudoUrls = shared.constructPseudoUrlInstances(pseudoUrlSources);+ const pseudoUrls = constructPseudoUrlInstances(pseudoUrlSources);expect(pseudoUrls).toHaveLength(4);pseudoUrls.forEach((purl) => {expect(purl.matches('https://example.com/foo')).toBe(true);@@ -22,8 +22,8 @@ describe('Enqueue links shared functions', () => {});test('should cache items', () => {- const pseudoUrls = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);- const pseudoUrls2 = shared.constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);+ const pseudoUrls = constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);+ const pseudoUrls2 = constructPseudoUrlInstances(['http[s?]://example.com/[.*]']);expect(pseudoUrls[0] === pseudoUrls2[0]).toBe(true);});});@@ -44,8 +44,8 @@ describe('Enqueue links shared functions', () => {return request;};- const requestOptions = shared.createRequestOptions(sources);- const requests = shared.createRequests(requestOptions, pseudoUrls).map(transformRequestFunction).filter((r) => !!r);+ const requestOptions = createRequestOptions(sources);+ const requests = createRequests(requestOptions, pseudoUrls).map(transformRequestFunction).filter((r) => !!r);expect(requests).toHaveLength(2);requests.forEach((r) => {@@ -67,7 +67,7 @@ describe('Enqueue links shared functions', () => {const requests = Array(5).fill(null).map((_, i) => i);- const finished = shared.addRequestsToQueueInBatches(requests, fakeRequestQueue, 2);+ const finished = addRequestsToQueueInBatches(requests, fakeRequestQueue, 2);// With batch size 2, two requests will be dispatched synchronously before the async function// returns and thus the following push should place 1000 on the third place in the array.diff --git a/test/storages/dataset.test.js b/test/storages/dataset.test.jsindex 782a87e7c..a927d0aa4 100644--- a/test/storages/dataset.test.js+++ b/test/storages/dataset.test.js@@ -8,7 +8,7 @@ import {checkAndSerialize,chunkBySize,} from '../../build/storages/dataset';-import * as Apify from '../../build';+import Apify from '../../build';import { StorageManager } from '../../build/storages/storage_manager';jest.mock('../../build/storages/storage_manager');diff --git a/test/storages/key_value_store.test.js b/test/storages/key_value_store.test.jsindex 2c40a3c22..8c6d9a859 100644--- a/test/storages/key_value_store.test.js+++ b/test/storages/key_value_store.test.js@@ -7,7 +7,7 @@ import {maybeStringify,} from '../../build/storages/key_value_store';import { StorageManager } from '../../build/storages/storage_manager';-import * as Apify from '../../build';+import Apify from '../../build';jest.mock('../../build/storages/storage_manager');diff --git a/test/storages/request_queue.test.js b/test/storages/request_queue.test.jsindex 324cb9961..e94c550d0 100644--- a/test/storages/request_queue.test.js+++ b/test/storages/request_queue.test.js@@ -1,4 +1,4 @@-import * as Apify from '../../build';+import Apify from '../../build';import { apifyClient } from '../../build/utils';import {RequestQueue,diff --git a/test/utils.test.js b/test/utils.test.jsindex bf5998664..accbf78c9 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -11,7 +11,7 @@ import Apify from '../build/index';import * as utils from '../build/utils';import log from '../build/utils_log';import * as requestUtils from '../build/utils_request';-import * as htmlToTextData from './data/html_to_text_test_data';+import htmlToTextData from './data/html_to_text_test_data';describe('utils.newClient()', () => {test('reads environment variables correctly', () => {diff --git a/tools/typescript_fixes.js b/tools/typescript_fixes.jsindex 030763ba0..b14609c9c 100644--- a/tools/typescript_fixes.js+++ b/tools/typescript_fixes.js@@ -1,4 +1,4 @@-const { readdirSync, readFileSync, writeFileSync, mkdirSync, realpathSync, statSync } = require('fs');+const { readdirSync, readFileSync, writeFileSync, realpathSync, statSync } = require('fs');const path = require('path');const debug = (namespace, ...args) => {@@ -10,12 +10,11 @@ const debug = (namespace, ...args) => {};console.log('Currect directory:', realpathSync('./'));-const typesPath = './types';+const typesPath = './build';const paths = {main: path.join(typesPath, 'main.d.ts'),session_pool: path.join(typesPath, 'session_pool', 'session_pool.d.ts'),utils_log: path.join(typesPath, 'utils_log.d.ts'),- playwright_utils: path.join(typesPath, 'playwright_utils.d.ts'),};console.log('Processing', paths.main);@@ -72,70 +71,6 @@ const fixUtilsLog = async () => {}};-const addTypeReference = (input, types) => {- return `${types.map((type) => `/// <reference path="${type}" />`).join('\n')}\n\n${input}`;-};--const padLeft = (input, length = 4) => {- return input.split('\n').map((line) => `${' '.repeat(length)}${line}`).join('\n');-};--const makePathsAbsolute = (input, root) => {- return input.split('\n').map((line) => {- return line.replace(/\.\//, `${root}/`);- }).join('\n');-};--const convertPlaywrightIndex = (input) => {- const output = makePathsAbsolute(input, 'playwright');- return addTypeReference(`declare module 'playwright' {\n${output}\n}`, ['./types/types.d.ts']);-};--const convertPlaywrightProtocol = (input) => {- const output = padLeft(makePathsAbsolute(input, 'playwright/types'));- return `declare module 'playwright/types/protocol' {\n${output}\n}`;-};--const convertPlaywrightStructs = (input) => {- const output = padLeft(makePathsAbsolute(input, 'playwright/types'));- return addTypeReference(`declare module 'playwright/types/structs' {\n${output}\n}`, ['./types.d.ts']);-};--const convertPlaywrightTypes = (input) => {- const output = padLeft(makePathsAbsolute(input, 'playwright/types'));- return addTypeReference(`declare module 'playwright/types/types' {\n${output}\n}`, ['./protocol.d.ts', './structs.d.ts']);-};--const inlinePlaywrightTypes = async () => {- // copy files to `types/playwright`- const files = [- { path: 'index.d.ts', convertor: convertPlaywrightIndex },- { path: 'types/protocol.d.ts', convertor: convertPlaywrightProtocol },- { path: 'types/structs.d.ts', convertor: convertPlaywrightStructs },- { path: 'types/types.d.ts', convertor: convertPlaywrightTypes },- ];- mkdirSync(path.join('types', 'playwright'));- mkdirSync(path.join('types', 'playwright', 'types'));-- for (const file of files) {- const fromPath = path.join('node_modules', 'playwright', file.path);- const toPath = path.join('types', 'playwright', file.path);- const input = readFileSync(fromPath, { encoding: 'utf8' });- const output = file.convertor(input);- console.log('Writing file', toPath);- writeFileSync(toPath, output, { encoding: 'utf8' });- }-- // add type references to playwright_utils.d.ts- const input = readFileSync(paths.playwright_utils, { encoding: 'utf8' });- const output = addTypeReference(input, [- '../types/playwright/index.d.ts',- '../types/playwright/types/types.d.ts',- ]);- console.log('Writing file', paths.playwright_utils);- writeFileSync(paths.playwright_utils, output, { encoding: 'utf8' });-};-/*** @callback FilenameFilter* @param {String} path Relative path to file considered.@@ -261,11 +196,7 @@ const fixTypesReferences = async () => {let match;for (const line of input) {/* eslint-disable no-cond-assign */- if (match = line.match(/\/\/\/\s*<reference\s*types="types-apify\/([^"]+)"\s*\/>/)) {- debug('fixTypesReferences', 'fixing types-apify from file', filepath);- output.push(`/// <reference path="../types-apify/${match[1]}.d.ts" />`);- changed = true;- } else if (match = line.match(/^([^"]+)"node\/([^$]+)/)) {+ if (match = line.match(/^([^"]+)"node\/([^$]+)/)) {debug('fixTypesReferences', 'fixing "node/" from file', filepath);output.push(`${match[1]} "${match[2]}`);changed = true;@@ -274,6 +205,11 @@ const fixTypesReferences = async () => {output.push('// @ts-ignore optional peer dependency');output.push(line);changed = true;+ } else if (match = line.match(/^([^"]+)"playwright[/"]/)) {+ debug('fixTypesReferences', 'fixing "playwright" from file', filepath);+ output.push('// @ts-ignore optional peer dependency');+ output.push(line);+ changed = true;} else {output.push(line);}@@ -292,5 +228,4 @@ const fixTypesReferences = async () => {await fixTypesReferences();await fixSessionPool();await fixUtilsLog();- await inlinePlaywrightTypes();})();diff --git a/tsconfig.json b/tsconfig.jsonindex 9f7c71317..c09189e59 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -1,41 +1,18 @@{+ "extends": "@apify/tsconfig","compilerOptions": {- "target": "es2018",- "module": "commonjs",- "moduleResolution": "node",+ "target": "ES2020","lib": ["DOM",- "DOM.Iterable",- "ES2015",- "ES2016",- "ES2018",- "ES2019.Object",- "ES2018.AsyncIterable",- "ES2020.String",- "ES2019.Array"+ "ES2020",],- "baseUrl": "./",+ "baseUrl": ".","allowJs": true,- "checkJs": false,- "noEmit": false,- "newLine": "lf",- "declaration": true,- "emitDeclarationOnly": false,- "strict": true,- "noImplicitAny": true,- "stripInternal": true,- "noImplicitReturns": true,- "noImplicitThis": true,- "alwaysStrict": true,- "noStrictGenericChecks": true,- "strictNullChecks": true,- "allowSyntheticDefaultImports": true,- "esModuleInterop": false,- "declarationDir": "types/",- "outDir": "build/"+ "resolveJsonModule": false,+ "noUnusedLocals": false,+ "outDir": "build"},"include": [- "./types-apify/**/*.d.ts",- "./src/**/*.js"+ "./src/**/*"]}diff --git a/types-apify/apify-http-request.d.ts b/types-apify/apify-http-request.d.tsdeleted file mode 100644index 8ed3a6efb..000000000--- a/types-apify/apify-http-request.d.ts+++ /dev/null@@ -1,8 +0,0 @@-declare module '@apify/http-request' {- const EXPORTED: (...args: any) => any;- export = EXPORTED;-}--declare module '@apify/http-request/src/errors' {- export const TimeoutError: Error;-}diff --git a/types-apify/ps-tree.d.ts b/types-apify/ps-tree.d.tsdeleted file mode 100644index 70dee81a3..000000000--- a/types-apify/ps-tree.d.ts+++ /dev/null@@ -1,4 +0,0 @@-declare module '@apify/ps-tree' {- const EXPORTED: (pid: any, includeRoot: any, callback: any) => void;- export = EXPORTED;-}