🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
ops: rewrite tests logic to cache downloaded browsers before testing
Karim shoaird603c15212b2
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 94d106a..4b35e88 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -10,33 +10,89 @@ concurrency:cancel-in-progress: truejobs:+ # Step 1: Install and cache browsers separately+ setup-browsers:+ runs-on: macos-latest+ outputs:+ cache-key: ${{ steps.browser-cache.outputs.cache-primary-key }}++ steps:+ - uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v5+ with:+ python-version: "3.11" # Use one version for browser setup+ cache: 'pip'++ # Cache browsers based on versions in dependencies+ - name: Cache browsers+ id: browser-cache+ uses: actions/cache@v4+ with:+ path: |+ ~/.cache/ms-playwright+ ~/.camoufox+ ~/Library/Caches/ms-playwright+ key: browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-playwright-1.52.0-camoufox-latest+ restore-keys: |+ browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-+ browsers-${{ runner.os }}-++ # Only install if the cache misses+ - name: Install browser dependencies+ if: steps.browser-cache.outputs.cache-hit != 'true'+ run: |+ python3 -m pip install --upgrade pip+ python3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox++ - name: Install browsers (with retry logic)+ if: steps.browser-cache.outputs.cache-hit != 'true'+ run: |+ # Retry logic for rate limiting+ for i in {1..3}; do+ echo "Attempt $i: Installing Chromium"+ if python3 -m playwright install chromium; then+ break+ fi+ echo "Attempt $i failed, waiting 30 seconds..."+ sleep 30+ done++ for i in {1..3}; do+ echo "Attempt $i: Installing dependencies"+ if python3 -m playwright install-deps chromium firefox; then+ break+ fi+ echo "Attempt $i failed, waiting 30 seconds..."+ sleep 30+ done++ for i in {1..3}; do+ echo "Attempt $i: Fetching Camoufox"+ if python3 -m camoufox fetch --browserforge; then+ break+ fi+ echo "Attempt $i failed, waiting 60 seconds..."+ sleep 60+ done++ # Verify browsers are installed+ - name: Verify browser installation+ run: |+ ls -la ~/.cache/ms-playwright/ || true+ ls -la ~/.camoufox/ || true+ echo "Browser setup completed successfully!"++ # Step 2: Run tests using cached browserstests:+ needs: setup-browserstimeout-minutes: 60- runs-on: ${{ matrix.os }}+ runs-on: macos-lateststrategy:fail-fast: falsematrix:- include:- - python-version: "3.9"- os: macos-latest- env:- TOXENV: py- - python-version: "3.10"- os: macos-latest- env:- TOXENV: py- - python-version: "3.11"- os: macos-latest- env:- TOXENV: py- - python-version: "3.12"- os: macos-latest- env:- TOXENV: py- - python-version: "3.13"- os: macos-latest- env:- TOXENV: py+ python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]steps:- uses: actions/checkout@v4@@ -51,27 +107,43 @@ jobs:requirements*.txttox.ini- - name: Install Camoufox Dependencies+ # Restore browsers from the cache (should always hit)+ - name: Restore browser cache+ uses: actions/cache/restore@v4+ with:+ path: |+ ~/.cache/ms-playwright+ ~/.camoufox+ ~/Library/Caches/ms-playwright+ key: browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-playwright-1.52.0-camoufox-latest+ restore-keys: |+ browsers-${{ runner.os }}-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}-+ browsers-${{ runner.os }}-++ - name: Install Python dependencies onlyrun: |python3 -m pip install --upgrade pippython3 -m pip install playwright==1.52.0 rebrowser-playwright==1.52.0 camoufox- python3 -m playwright install chromium- python3 -m playwright install-deps chromium firefox- python3 -m camoufox fetch --browserforge+ # No browser installation here - using cached browsers!# Cache tox environments- name: Cache tox environments- uses: actions/cache@v3+ uses: actions/cache@v4with:path: .tox- # Include python version and os in cache keykey: tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('tox.ini', 'pyproject.toml', 'requirements*.txt') }}restore-keys: |tox-v1-${{ runner.os }}-py${{ matrix.python-version }}-tox-v1-${{ runner.os }}-+ - name: Verify browsers are available+ run: |+ ls -la ~/.cache/ms-playwright/ || echo "Playwright cache not found"+ ls -la ~/.camoufox/ || echo "Camoufox cache not found"+- name: Run tests- env: ${{ matrix.env }}+ env:+ TOXENV: pyrun: |pip install -U tox- tox+ tox\ No newline at end of file