🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
ops: new approach to CI tests workflow
Karim shoairac3db69c47fa
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 6ec968d..82e234d 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -10,88 +10,33 @@ concurrency:cancel-in-progress: truejobs:- # Step 1: Install and cache browsers separately- setup-browsers:- runs-on: macos-latest- outputs:- cache-key: ${{ steps.browser-cache.outputs.cache-primary-key }}-- steps:- - uses: actions/checkout@v4-- - name: Set up Python- uses: actions/setup-python@v5- with:- python-version: "3.11" # Use one version for browser setup- cache: 'pip'-- # Cache browsers based on versions in dependencies- - name: Cache browsers- id: browser-cache- uses: actions/cache@v4- with:- path: |- ~/Library/Caches/camoufox- ~/Library/Caches/ms-playwright- key: browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-playwright-1.52.0-camoufox-latest- restore-keys: |- browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-- browsers-${{ runner.os }}--- # Only install if the cache misses- - name: Install browser dependencies- if: steps.browser-cache.outputs.cache-hit != 'true'- run: |- python3 -m pip install --upgrade pip- python3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox-- - name: Install browsers (with retry logic)- if: steps.browser-cache.outputs.cache-hit != 'true'- run: |- # Retry logic for rate limiting- for i in {1..3}; do- echo "Attempt $i: Installing Chromium"- if python3 -m playwright install chromium; then- break- fi- echo "Attempt $i failed, waiting 30 seconds..."- sleep 30- done-- for i in {1..3}; do- echo "Attempt $i: Installing dependencies"- if python3 -m playwright install-deps chromium firefox; then- break- fi- echo "Attempt $i failed, waiting 30 seconds..."- sleep 30- done-- for i in {1..3}; do- echo "Attempt $i: Fetching Camoufox"- if python3 -m camoufox fetch --browserforge; then- break- fi- echo "Attempt $i failed, waiting 60 seconds..."- sleep 60- done-- # Verify browsers are installed- - name: Verify browser installation- run: |- ls -la ~/Library/Caches/ms-playwright/ || true- ls -la ~/Library/Caches/camoufox/ || true- echo "Browser setup completed successfully!"-- # Step 2: Run tests using cached browserstests:- needs: setup-browserstimeout-minutes: 60- runs-on: macos-latest+ runs-on: ${{ matrix.os }}strategy:fail-fast: falsematrix:- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]+ include:+ - python-version: "3.9"+ os: macos-latest+ env:+ TOXENV: py39+ - python-version: "3.10"+ os: macos-latest+ env:+ TOXENV: py310+ - python-version: "3.11"+ os: macos-latest+ env:+ TOXENV: py311+ - python-version: "3.12"+ os: macos-latest+ env:+ TOXENV: py312+ - python-version: "3.13"+ os: macos-latest+ env:+ TOXENV: py313steps:- uses: actions/checkout@v4@@ -106,42 +51,32 @@ jobs:requirements*.txttox.ini- # Restore browsers from the cache (should always hit)- - name: Restore browser cache- uses: actions/cache/restore@v4- with:- path: |- ~/Library/Caches/camoufox- ~/Library/Caches/ms-playwright- key: browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-playwright-1.52.0-camoufox-latest- restore-keys: |- browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-- browsers-${{ runner.os }}--- - name: Install Python dependencies only+ # Install browsers ONCE at the workflow level+ - name: Install browser dependenciesrun: |python3 -m pip install --upgrade pippython3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox- # No browser installation here - using cached browsers!++ - name: Install browsers+ run: |+ python3 -m playwright install chromium+ python3 -m playwright install-deps chromium firefox+ python3 -m camoufox fetch --browserforge# Cache tox environments- name: Cache tox environmentsuses: actions/cache@v4with:path: .tox+ # Include python version and os in cache keykey: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'pyproject.toml', 'requirements*.txt') }}restore-keys: |tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-tox-v1-${{ runner.os }}-- - name: Verify browsers are available- run: |- ls -la ~/Library/Caches/ms-playwright/ || echo "Playwright cache not found"- ls -la ~/Library/Caches/camoufox/ || echo "Camoufox cache not found"+ - name: Install tox+ run: pip install -U tox- name: Run tests- env:- TOXENV: py- run: |- pip install -U tox- tox\ No newline at end of file+ env: ${{ matrix.env }}+ run: tox\ No newline at end of filediff --git a/tox.ini b/tox.iniindex 07b2831..091642b 100644--- a/tox.ini+++ b/tox.ini@@ -10,11 +10,11 @@ envlist = pre-commit,py{39,310,311,312,313}usedevelop = Truechangedir = testsdeps =+ playwright==1.52.0+ rebrowser-playwright==1.52.0+ camoufox-r{toxinidir}/tests/requirements.txtcommands =- playwright install chromium- playwright install-deps chromium firefox- camoufox fetch --browserforge# Test async tests without parallelization to escape Github CI issues with nested loopspytest --cov=scrapling --cov-report=xml -m "asyncio" --verbosepytest --cov=scrapling --cov-report=xml -m "not asyncio" -n auto --cov-append