Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: fix support for building TypeScript projects
Martin Adámekb1fb02015450
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 25c9c95d2..bc85f98d4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -46,6 +46,7 @@ await dataset.pushData({ myValue: 123 });- Fix `proxyUrl` without a port throwing an error when launching browsers.- Fix `maxUsageCount` of a `Session` not being persisted.- Update `puppeteer` and `playwright` to match stable Chrome (90).+- Fix support for building TypeScript projects that depend on the SDK.1.2.1 / 2021/05/14====================diff --git a/src/apify.js b/src/apify.jsindex c8e1c7362..3db78196f 100644--- a/src/apify.js+++ b/src/apify.js@@ -20,14 +20,6 @@ import { enqueueLinks } from './enqueue_links/enqueue_links';import { requestAsBrowser } from './utils_request';import { ApifyCallError } from './errors';-/**- * @typedef AdhocWebhook- * @property {EventTypes} eventTypes- * @property {string} requestUrl- * @property {string} [idempotencyKey]- * @property {string} [payloadTemplate]- */-/*** `Apify` class serves as an alternative approach to the static helpers exported from the package. It allows to pass configuration* that will be used on the instance methods. Environment variables will have precedence over this configuration.@@ -452,22 +444,6 @@ export class Apify {await sleep(customAfterSleepMillis || this.config.get('metamorphAfterSleepMillis'));}- /**- * @typedef WebhookRun- * @property {string} id- * @property {string} createdAt- * @property {string} modifiedAt- * @property {string} userId- * @property {boolean} isAdHoc- * @property {EventTypes} eventTypes- * @property {*} condition- * @property {boolean} ignoreSslErrors- * @property {boolean} doNotRetry- * @property {string} requestUrl- * @property {string} payloadTemplate- * @property {*} lastDispatch- * @property {*} stats- *//**** Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed.diff --git a/src/utils_log.js b/src/utils_log.jsindex 7f7d93f57..21dce779f 100644--- a/src/utils_log.js+++ b/src/utils_log.js@@ -11,9 +11,9 @@* @property {number} [maxStringLength=2000] Max length of the string to be logged. Longer strings will be truncated.* @property {string} [prefix] Prefix to be prepended the each logged line.* @property {string} [suffix] Suffix that will be appended the each logged line.- * @property {object} [logger] Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable+ * @property {Logger} [logger] Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable* strings. Optionally you can use `log.LoggerJson` that formats each log line as a JSON.- * @property {object} [data] Additional data to be added to each log line.+ * @property {Record<string, unknown>} [data] Additional data to be added to each log line.*//**diff --git a/tools/typescript_fixes.js b/tools/typescript_fixes.jsindex 0706cf700..297aec1d4 100644--- a/tools/typescript_fixes.js+++ b/tools/typescript_fixes.js@@ -1,11 +1,10 @@-const { readdirSync, readFileSync, writeFileSync, realpathSync, statSync } = require('fs');+const { readdirSync, readFileSync, writeFileSync, mkdirSync, realpathSync, statSync } = require('fs');const path = require('path');const debug = (namespace, ...args) => {- if (- process.env.DEBUG_TYPEFIXES- && (process.env.DEBUG_TYPEFIXES === '*' || process.env.DEBUG_TYPEFIXES.includes(namespace))- ) {+ const debugTypefixes = process.env.DEBUG_TYPEFIXES;++ if (debugTypefixes && (debugTypefixes === '*' || debugTypefixes.includes(namespace))) {console.log(...args);}};@@ -15,59 +14,14 @@ const typesPath = './types';const paths = {main: path.join(typesPath, 'main.d.ts'),session_pool: path.join(typesPath, 'session_pool', 'session_pool.d.ts'),+ utils_log: path.join(typesPath, 'utils_log.d.ts'),+ playwright_utils: path.join(typesPath, 'playwright_utils.d.ts'),};-console.log('Processing', paths['main']);--/**- * Removes `namespace log` and adds appropriate import and reference to main.d.ts.- *- * @param {string[]} input- * @return {string[]}- */-const fixLog = (input) => {- const output = [];-- const modes = {- base: 0, // Most reading and writing is in `base` mode: keep each line as is.- 'log-ns-replace': 1, // Erasing log namespace declaration and adding proper import- };-- let index = 0;- let mode = modes.base;- for (const line of input) {- try {- if (mode === modes.base) {- // if (index === 0) {- // output.push('/// <reference path="../src/utils_log.d.ts" />');- // debug('fixLog', 'line #1', mode);- // }- if (line.match(/^declare namespace log {/)) {- mode = modes['log-ns-replace'];- debug('fixLog', 'switch to', mode, line);- } else {- output.push(line);- debug('fixLog', 'out', line);- }- } else if (mode === modes['log-ns-replace']) {- if (line.match(/^\}/)) {- mode = modes.base;- debug('fixLog', 'switch to', mode, line);- } else {- // nothing- debug('fixLog', 'remove ', line);- }- }- } finally {- index++;- }- }- return output;-};+console.log('Processing', paths.main);const fixSessionPool = async () => {- const input = readFileSync(paths.session_pool, { encoding: 'utf8' })- .split('\n');+ const input = readFileSync(paths.session_pool, { encoding: 'utf8' }).split('\n');const matchLines = [/addListener\(event/,@@ -84,7 +38,7 @@ const fixSessionPool = async () => {let changed = false;for (const line of input) {- if (!matchLines.some(s => s.test(line))) {+ if (!matchLines.some((s) => s.test(line))) {output.push(line);} else {changed = true;@@ -98,6 +52,90 @@ const fixSessionPool = async () => {}};+const fixUtilsLog = async () => {+ const input = readFileSync(paths.utils_log, { encoding: 'utf8' }).split('\n');+ const output = [];+ let changed = false;++ for (const line of input) {+ if (!line.includes('import { LoggerOptions } from "@apify/log/log"')) {+ output.push(line);+ } else {+ changed = true;+ debug('fixUtilsLog', 'removing line', line);+ }+ }++ if (changed) {+ console.log('Writing file', paths.utils_log);+ writeFileSync(paths.utils_log, output.join('\n'), { encoding: 'utf8' });+ }+};++const addTypeReference = (input, types) => {+ return `${types.map((type) => `/// <reference path="${type}" />`).join('\n')}\n\n${input}`;+};++const padLeft = (input, length = 4) => {+ return input.split('\n').map((line) => `${' '.repeat(length)}${line}`).join('\n');+};++const makePathsAbsolute = (input, root) => {+ return input.split('\n').map((line) => {+ return line.replace(/\.\//, `${root}/`);+ }).join('\n');+};++const convertPlaywrightIndex = (input) => {+ const output = makePathsAbsolute(input, 'playwright');+ return addTypeReference(`declare module 'playwright' {\n${output}\n}`, ['./types/types.d.ts']);+};++const convertPlaywrightProtocol = (input) => {+ const output = padLeft(makePathsAbsolute(input, 'playwright/types'));+ return `declare module 'playwright/types/protocol' {\n${output}\n}`;+};++const convertPlaywrightStructs = (input) => {+ const output = padLeft(makePathsAbsolute(input, 'playwright/types'));+ return addTypeReference(`declare module 'playwright/types/structs' {\n${output}\n}`, ['./types.d.ts']);+};++const convertPlaywrightTypes = (input) => {+ const output = padLeft(makePathsAbsolute(input, 'playwright/types'));+ return addTypeReference(`declare module 'playwright/types/types' {\n${output}\n}`, ['./protocol.d.ts', './structs.d.ts']);+};++const inlinePlaywrightTypes = async () => {+ // copy files to `types/playwright`+ const files = [+ { path: 'index.d.ts', convertor: convertPlaywrightIndex },+ { path: 'types/protocol.d.ts', convertor: convertPlaywrightProtocol },+ { path: 'types/structs.d.ts', convertor: convertPlaywrightStructs },+ { path: 'types/types.d.ts', convertor: convertPlaywrightTypes },+ ];+ mkdirSync(path.join('types', 'playwright'));+ mkdirSync(path.join('types', 'playwright', 'types'));++ for (const file of files) {+ const fromPath = path.join('node_modules', 'playwright', file.path);+ const toPath = path.join('types', 'playwright', file.path);+ const input = readFileSync(fromPath, { encoding: 'utf8' });+ const output = file.convertor(input);+ console.log('Writing file', toPath);+ writeFileSync(toPath, output, { encoding: 'utf8' });+ }++ // add type references to playwright_utils.d.ts+ const input = readFileSync(paths.playwright_utils, { encoding: 'utf8' });+ const output = addTypeReference(input, [+ '../types/playwright/index.d.ts',+ '../types/playwright/types/types.d.ts',+ ]);+ console.log('Writing file', paths.playwright_utils);+ writeFileSync(paths.playwright_utils, output, { encoding: 'utf8' });+};+/*** @callback FilenameFilter* @param {String} path Relative path to file considered.@@ -138,7 +176,8 @@ const traverseDirs = async (dir, filter, handleFile) => {return types;};-const EXCLUDED_FROM_EXPORT = [/^Session$/, /^RequestList$/, /Local$/];+const EXCLUDED_CLASSES = ['Session', 'RequestList', 'Apify', 'Configuration', 'Configuration', 'BasicCrawler', 'LoggerOptions'];+const EXCLUDED_FROM_EXPORT = [new RegExp(`^(${EXCLUDED_CLASSES.join('|')})$`), /Local$/];/*** Blindly extracts exported typenames from a `*.d.ts` file.@@ -150,9 +189,9 @@ const readTypes = (filepath) => {const inputByLine = input.split('\n');const types = [];for (const line of inputByLine) {- const matches = line.match(/^export (?:type|class) ([^<\s]+)/); // get exported types without generic Export<T>+ const matches = line.match(/^export (?:type|class|interface) ([^<\s]+)/); // get exported types without generic Export<T>debug('readTypes', !!matches, line);- if (matches && !EXCLUDED_FROM_EXPORT.some(s => s.test(matches[1]))) {+ if (matches && !EXCLUDED_FROM_EXPORT.some((s) => s.test(matches[1]))) {types.push(matches[1]);}}@@ -206,54 +245,47 @@ const processIndexEsm = async () => {},readTypes,);- const input = readFileSync(paths['main'], { encoding: 'utf8' })+ const input = readFileSync(paths.main, { encoding: 'utf8' }).split('\n');- console.log('Fixing log module', paths['main']);- // const fixedLog = fixLog(input);- console.log('Fixing type exports', paths['main']);+ console.log('Fixing type exports', paths.main);const fixedTypes = fixTypes(input, types);- console.log('Writing', paths['main']);- writeFileSync(paths['main'], fixedTypes.join('\n'));+ console.log('Writing', paths.main);+ writeFileSync(paths.main, fixedTypes.join('\n'));};const fixTypesReferences = async () => {- await traverseDirs(- typesPath,- (filename) => {- return filename.endsWith('.d.ts');- },- (filepath) => {- const input = readFileSync(filepath, { encoding: 'utf8' })- .split('\n');- const output = [];- let changed = false;- let match;- for (const line of input) {- /* eslint-disable no-cond-assign */- if (match = line.match(/\/\/\/\s*<reference\s*types="types-apify\/([^"]+)"\s*\/>/)) {- debug('fixTypesReferences', 'fixing types-apify from file', filepath);- output.push(`/// <reference path="../types-apify/${match[1]}.d.ts" />`);- changed = true;- } else if (match = line.match(/^([^"]+)"node\/([^$]+)/)) {- debug('fixTypesReferences', 'fixing "node/" from file', filepath);- output.push(`${match[1]} "${match[2]}`);- changed = true;- } else {- output.push(line);- }- /* eslint-enable no-cond-assign */+ await traverseDirs(typesPath, (filename) => filename.endsWith('.d.ts'), (filepath) => {+ const input = readFileSync(filepath, { encoding: 'utf8' }).split('\n');+ const output = [];+ let changed = false;+ let match;+ for (const line of input) {+ /* eslint-disable no-cond-assign */+ if (match = line.match(/\/\/\/\s*<reference\s*types="types-apify\/([^"]+)"\s*\/>/)) {+ debug('fixTypesReferences', 'fixing types-apify from file', filepath);+ output.push(`/// <reference path="../types-apify/${match[1]}.d.ts" />`);+ changed = true;+ } else if (match = line.match(/^([^"]+)"node\/([^$]+)/)) {+ debug('fixTypesReferences', 'fixing "node/" from file', filepath);+ output.push(`${match[1]} "${match[2]}`);+ changed = true;+ } else {+ output.push(line);}+ /* eslint-enable no-cond-assign */+ }- if (changed === true) {- console.log('Writing', filepath);- writeFileSync(filepath, output.join('\n'));- }- },- );+ if (changed === true) {+ console.log('Writing', filepath);+ writeFileSync(filepath, output.join('\n'));+ }+ });};(async () => {await processIndexEsm();await fixTypesReferences();await fixSessionPool();+ await fixUtilsLog();+ await inlinePlaywrightTypes();})();diff --git a/types-apify/apify-shared/log.d.ts b/types-apify/apify-shared/log.d.tsdeleted file mode 100644index dfd40d793..000000000--- a/types-apify/apify-shared/log.d.ts+++ /dev/null@@ -1,195 +0,0 @@-declare module 'apify-shared/log' {- export interface LoggerOptions {- /**- * Sets the log level to the given value, preventing messages from less important log levels- * from being printed to the console. Use in conjunction with the `log.LEVELS` constants.- */- level: number;- /**- * Max depth of data object that will be logged. Anything deeper than the limit will be stripped off.- */- maxDepth: number;- /**- * Max length of the string to be logged. Longer strings will be truncated.- */- maxStringLength: number;- /**- * Prefix to be prepended the each logged line.- */- prefix: string;- /**- * Suffix that will be appended the each logged line.- */- suffix: string;- /**- * Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable- * strings. Optionally you can use `log.LoggerJson` that formats each log line as a JSON.- */- logger: any;- /**- * Additional data to be added to each log line.- */- data: any;- }-- /**- * The log instance enables level aware logging of messages and we advise- * to use it instead of `console.log()` and its aliases in most development- * scenarios.- *- * A very useful use case for `log` is using `log.debug` liberally throughout- * the codebase to get useful logging messages only when appropriate log level is set- * and keeping the console tidy in production environments.- *- * The available logging levels are, in this order: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `OFF`- * and can be referenced from the `log.LEVELS` constant, such as `log.LEVELS.ERROR`.- *- * To log messages to the system console, use the `log.level(message)` invocation,- * such as `log.debug('this is a debug message')`.- *- * To prevent writing of messages above a certain log level to the console, simply- * set the appropriate level. The default log level is `INFO`, which means that- * `DEBUG` messages will not be printed, unless enabled.- *- * **Example:**- * ```- * const Apify = require('apify');- * const { log } = Apify.utils;- *- * log.info('Information message', { someData: 123 }); // prints message- * log.debug('Debug message', { debugData: 'hello' }); // doesn't print anything- *- * log.setLevel(log.LEVELS.DEBUG);- * log.debug('Debug message'); // prints message- *- * log.setLevel(log.LEVELS.ERROR);- * log.debug('Debug message'); // doesn't print anything- * log.info('Info message'); // doesn't print anything- *- * log.error('Error message', { errorDetails: 'This is bad!' }); // prints message- * try {- * throw new Error('Not good!');- * } catch (e) {- * log.exception(e, 'Exception occurred', { errorDetails: 'This is really bad!' }); // prints message- * }- *- * log.setOptions({ prefix: 'My actor' });- * log.info('I am running!'); // prints "My actor: I am running"- *- * const childLog = log.child({ prefix: 'Crawler' });- * log.info('I am crawling!'); // prints "My actor:Crawler: I am crawling"- * ```- *- * Another very useful way of setting the log level is by setting the `APIFY_LOG_LEVEL`- * environment variable, such as `APIFY_LOG_LEVEL=DEBUG`. This way, no code changes- * are necessary to turn on your debug messages and start debugging right away.- */- export class Log {- deprecationsReported: any;- options: LoggerOptions;- constructor(options: Partial<LoggerOptions>);- getLevel(): LEVELS;- setLevel(level: LEVELS): void;- internal(level: LEVELS, message: string, data: any, exception: any): void;- setOptions(options: Partial<LoggerOptions>): void;- getOptions(): LoggerOptions;- child(options: Partial<LoggerOptions>): Log;- error(message: string, data?: any): void;- exception(exception: Error, message?: string, data?: any): void;- softFail(message: string, data?: any): void;- warning(message: string, data?: any): void;- info(message: string, data?: any): void;- debug(message: string, data?: any): void;- perf(message: string, data?: any): void;- deprecated(message: string): void;- }-- export const LoggerText: any;- export const LoggerJson: any;-- /**- * Map of available log levels that's useful for easy setting of appropriate log levels.- * Each log level is represented internally by a number. Eg. `log.LEVELS.DEBUG === 5`.- */- export enum LEVELS {- // Turns off logging completely- OFF = 0,- // For unexpected errors in Apify system- ERROR = 1,- // For situations where error is caused by user (e.g. Meteor.Error), i.e. when the error is not- // caused by Apify system, avoid the word "ERROR" to simplify searching in log- SOFT_FAIL = 2,- WARNING = 3,- INFO = 4,- DEBUG = 5,- // for performance stats- PERF = 6- }-- // The shape of the Log class can't be easily typed in TS because it- // dynamically attaches values to the log instance, so we need to document- // only the "inner" functions of the Log class, and not the class itself- // That's why the export consts below:-- /**- * Returns the currently selected logging level. This is useful for checking whether a message- * will actually be printed to the console before one actually performs a resource intensive operation- * to construct the message, such as querying a DB for some metadata that need to be added. If the log- * level is not high enough at the moment, it doesn't make sense to execute the query.- */- export const getLevel: Log['getLevel'];- /**- * Sets the log level to the given value, preventing messages from less important log levels- * from being printed to the console. Use in conjunction with the `log.LEVELS` constants such as- *- * ```- * log.setLevel(log.LEVELS.DEBUG);- * ```- *- * Default log level is INFO.- */- export const setLevel: Log['setLevel'];- /**- * Configures logger.- */- export const setOptions: Log['setOptions'];- /**- * Returns the logger configuration.- */- export const getOptions: Log['getOptions'];- /**- * Creates a new instance of logger that inherits settings from a parent logger.- */- export const child: Log['child'];- /**- * Logs an `ERROR` message. Use this method to log error messages that are not directly connected- * to an exception. For logging exceptions, use the `log.exception` method.- */- export const error: Log['error'];- /**- * Logs an `ERROR` level message with a nicely formatted exception. Note that the exception is the first parameter- * here and an additional message is only optional.- */- export const exception: Log['exception'];- /**- * Logs a `WARNING` level message. Data are stringified and appended to the message.- */- export const warning: Log['warning'];- /**- * Logs an `INFO` message. `INFO` is the default log level so info messages will be always logged,- * unless the log level is changed. Data are stringified and appended to the message.- */- export const info: Log['info'];- /**- * Logs a `DEBUG` message. By default, it will not be written to the console. To see `DEBUG`- * messages in the console, set the log level to `DEBUG` either using the `log.setLevel(log.LEVELS.DEBUG)`- * method or using the environment variable `APIFY_LOG_LEVEL=DEBUG`. Data are stringified and appended- * to the message.- */- export const debug: Log['debug'];- /**- * Logs given message only once as WARNING. It's used to warn user that some feature he is using- * has been deprecated.- */- export const deprecated: Log['deprecated'];-}