Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Improve docs build script

Ondra Urban35a5a3c7894b
diff --git a/website/package.json b/website/package.json
index dc7e51582..0395f2290 100644
--- a/website/package.json
+++ b/website/package.json
@@ -15,7 +15,8 @@
"decamelize": "^4.0.0",
"docusaurus": "^1.13.0",
"jsdoc-to-markdown": "^5.0.1",
- "prettier": "^1.18.2"
+ "prettier": "^1.18.2",
+ "fs-extra": "^8.1.0"
},
"dependencies": {
"remarkable-extlink": "^1.1.0"
diff --git a/website/tools/build_docs.js b/website/tools/build_docs.js
index b39e47a85..7b5e3f58e 100644
--- a/website/tools/build_docs.js
+++ b/website/tools/build_docs.js
@@ -1,13 +1,11 @@
const decamelize = require('decamelize');
-const fs = require('fs');
+const fs = require('fs-extra');
const jsdoc2md = require('jsdoc-to-markdown'); // eslint-disable-line
const path = require('path');
const prettier = require('prettier'); // eslint-disable-line
const readline = require('readline');
-const { promisify } = require('util');
const prettierConfig = require('./prettier.config');
-const writeFile = promisify(fs.writeFile);
const BASE_URL = '/docs';
/* eslint-disable no-shadow */
@@ -156,7 +154,7 @@ const main = async () => {
const title = filename.split('.')[0].split('_').map(word => `${word[0].toUpperCase()}${word.substr(1)}`).join(' ');
const header = getHeader(title);
const markdown = prettier.format(`${header}\n${description}\n${codeblock}`, prettierConfig);
- await writeFile(path.join(exampleFilesOutputDir, `${title.replace(/\s/g, '')}.md`), markdown);
+ await fs.outputFile(path.join(exampleFilesOutputDir, `${title.replace(/\s/g, '')}.md`), markdown);
});
await Promise.all(examplePromises);
@@ -197,34 +195,36 @@ const main = async () => {
const output = jsdoc2md.renderSync(getRenderOptions(template, finalData));
const markdown = generateFinalMarkdown(mainModule, output, entityMap);
- fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${mainModule}.md`), markdown);
+ await fs.outputFile(path.resolve(sourceFilesOutputDir, `${mainModule}.md`), markdown);
// create a doc file file for each class
- classNames.forEach((className) => {
+ const cPs = classNames.map(async (className) => {
const template = `{{#class name="${className}"}}{{>docs}}{{/class}}`;
console.log(`Rendering ${className}, template: ${template}`); // eslint-disable-line no-console
const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
const markdown = generateFinalMarkdown(className, output, entityMap);
- fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${className}.md`), markdown);
+ await fs.outputFile(path.resolve(sourceFilesOutputDir, `${className}.md`), markdown);
});
// create a doc file file for each namespace
- namespaces.forEach((name) => {
+ const nPs = namespaces.map(async (name) => {
const template = `{{#namespace name="${name}"}}{{>docs}}{{/namespace}}`;
console.log(`Rendering ${name}, template: ${template}`); // eslint-disable-line no-console
const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
const markdown = generateFinalMarkdown(name, output, entityMap);
- fs.writeFileSync(path.resolve(sourceFilesOutputDir, `${name}.md`), markdown);
+ await fs.outputFile(path.resolve(sourceFilesOutputDir, `${name}.md`), markdown);
});
// create a doc file for each type
- typedefs.forEach((name) => {
+ const tPs = typedefs.map(async (name) => {
const template = `{{#identifier name="${name}"}}{{>docs}}{{/identifier}}`;
console.log(`Rendering ${name}, template: ${template}`); // eslint-disable-line no-console
const output = jsdoc2md.renderSync(getRenderOptions(template, templateData));
const markdown = generateFinalMarkdown(name, output, entityMap);
- fs.writeFileSync(path.resolve(typeFilesOutputDir, `${name}.md`), markdown);
+ await fs.outputFile(path.resolve(typeFilesOutputDir, `${name}.md`), markdown);
});
+
+ await Promise.all([...cPs, ...nPs, ...tPs]);
};
-main();
+main().then(() => console.log('All docs built succesfully.'));