Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: add development support on windows and tests to CI (#868)

Ondra Urban90b0f554a8bd
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 7664efe0b..5c68461ba 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -10,11 +10,12 @@ jobs:
test:
name: Test
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
+ os: [ubuntu-latest, windows-latest]
node-version: [ 10, 12, 14, 15, 16 ]
steps:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c1df0d836..f1412cc2e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -14,10 +14,11 @@ jobs:
build_and_test:
name: Build & Test
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.os }}
strategy:
matrix:
+ os: [ubuntu-latest, windows-latest]
node-version: [10, 12, 14, 15, 16]
steps:
@@ -36,7 +37,7 @@ jobs:
node-version: ${{ matrix.node-version }}
-
name: Cache Node Modules
- if: ${{ matrix.node-version == 14 }}
+ if: ${{ matrix.node-version == 14 && matrix.os == 'ubuntu-latest' }}
uses: actions/cache@v2
with:
path: |
diff --git a/package.json b/package.json
index e87295603..bb81d0684 100644
--- a/package.json
+++ b/package.json
@@ -46,7 +46,7 @@
"test": "npm run build && jest --silent",
"prepare": "npm run build",
"prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
- "clean": "rm -rf ./build ./types",
+ "clean": "rimraf ./build ./types",
"lint": "eslint ./src ./test",
"lint:fix": "eslint ./src ./test --ext .js,.jsx --fix"
},
@@ -75,7 +75,6 @@
"jquery": "^3.5.1",
"mime-types": "^2.1.27",
"ow": "^0.23.0",
- "rimraf": "^3.0.2",
"semver": "^7.3.2",
"socket.io": "^3.0.3",
"stream-json": "^1.7.1",
@@ -114,6 +113,7 @@
"prettier": "^2.1.2",
"proxy": "^1.0.2",
"puppeteer": "9.1.1",
+ "rimraf": "^3.0.2",
"sinon": "^9.2.0",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^3.0.3",
diff --git a/test/data/html_to_text_test.txt b/test/data/html_to_text_test.txt
deleted file mode 100644
index 28f225cb8..000000000
--- a/test/data/html_to_text_test.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Let's start with a simple text.
-The ships hung in the sky, much the way that bricks don't.
-These aren't the Droids you're looking for
-I'm sorry, Dave. I'm afraid I can't do that.
-I'm sorry, Dave. I'm afraid I can't do that.
-A1 A2 A3
-B1 B2 B3 B 4
-This is some text with inline elements and HTML entities (>bla<)
-Test
-a
-few
-line
-breaks
-Spaces in an inline text should be completely ignored.
-But,
- a pre-formatted
- block should be kept
- pre-formatted.
-The Greatest Science Fiction Quotes Of All Time
-Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.
\ No newline at end of file
diff --git a/test/data/html_to_text_test.html b/test/data/html_to_text_test_data.js
similarity index 62%
rename from test/data/html_to_text_test.html
rename to test/data/html_to_text_test_data.js
index 87c56a06a..d5d598074 100644
--- a/test/data/html_to_text_test.html
+++ b/test/data/html_to_text_test_data.js
@@ -1,4 +1,31 @@
-<html>
+/* eslint-disable */
+
+// The whitespace in the text is important. Don't change it.
+// We're keeping this text as a JS string, because git and other
+// tools do magic with line endings and it can break tests.
+// E.g. LF -> CRLF on Win or auto-trimming of lines in editors.
+exports.text = 'Let\'s start with a simple text. \n' +
+ 'The ships hung in the sky, much the way that bricks don\'t. \n' +
+ 'These aren\'t the Droids you\'re looking for\n' +
+ 'I\'m sorry, Dave. I\'m afraid I can\'t do that.\n' +
+ 'I\'m sorry, Dave. I\'m afraid I can\'t do that.\n' +
+ 'A1\tA2\tA3\t\n' +
+ 'B1\tB2\tB3\tB 4\t\n' +
+ 'This is some text with inline elements and HTML entities (>bla<) \n' +
+ 'Test\n' +
+ 'a\n' +
+ 'few\n' +
+ 'line\n' +
+ 'breaks\n' +
+ 'Spaces in an inline text should be completely ignored. \n' +
+ 'But,\n' +
+ ' a pre-formatted\n' +
+ ' block should be kept\n' +
+ ' pre-formatted.\n' +
+ 'The Greatest Science Fiction Quotes Of All Time \n' +
+ 'Don\'t know, I don\'t know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.'
+
+exports.html = `<html>
<head>
<title>Title SHOULD NOT be converted</title>
@@ -92,4 +119,4 @@ But,
just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
</p>
</body>
-</html>
+</html>`
diff --git a/test/utils.test.js b/test/utils.test.js
index ad9b5adc1..ad7d12368 100644
--- a/test/utils.test.js
+++ b/test/utils.test.js
@@ -11,6 +11,7 @@ import Apify from '../build/index';
import * as utils from '../build/utils';
import log from '../build/utils_log';
import * as requestUtils from '../build/utils_request';
+import * as htmlToTextData from './data/html_to_text_test_data';
describe('utils.newClient()', () => {
test('reads environment variables correctly', () => {
@@ -550,11 +551,9 @@ describe('utils.htmlToText()', () => {
});
test('handles larger HTML documents', () => {
- const html1 = fs.readFileSync(path.join(__dirname, 'data', 'html_to_text_test.html'), 'utf8');
- const text1 = fs.readFileSync(path.join(__dirname, 'data', 'html_to_text_test.txt'), 'utf8');
-
+ const { html, text } = htmlToTextData;
// Careful here - don't change any whitespace in the text below or the test will break, even trailing!
- checkHtmlToText(html1, text1, true);
+ checkHtmlToText(html, text, true);
});
test('works with Cheerio object', () => {