Scrapy, a fast high-level web crawling & scraping framework for Python.
Add VCS CI job, fix support for upcoming parsel version (#7924)
Adrian94dff69468a9
diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.ymlnew file mode 100644index 000000000..bf867dba7--- /dev/null+++ b/.github/workflows/tests-vcs-deps.yml@@ -0,0 +1,53 @@+name: VCS dependencies++permissions:+ contents: read++on:+ schedule:+ - cron: '0 4 * * *'+ workflow_dispatch:++concurrency:+ group: ${{github.workflow}}-${{ github.ref }}+ cancel-in-progress: true++jobs:+ tests:+ name: tests+ runs-on: ubuntu-latest+ env:+ PYTEST_ADDOPTS: -n auto --no-cov+ TOXENV: vcs-deps+ UV_PYTHON_PREFERENCE: only-system+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Set up Python+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.14"++ # Dependencies that ship wheels on PyPI are built from source here, so+ # their build dependencies are needed: libxml2 and libxslt for lxml,+ # libjpeg and zlib for Pillow, and autotools for the libuv bundled in+ # uvloop.+ - name: Install system libraries+ run: |+ sudo apt-get update+ sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev++ - name: Set up uv+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini++ - name: Install mitmproxy+ run: uv tool install --python cpython mitmproxy++ - name: Run tests+ run: uvx --with tox-uv toxdiff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex f6334c32c..fa91e2904 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref):``"json"``, ``"text"`` or ``None`` (default). It's passed to:class:`parsel.Selector` and its meaning is defined there. However, when``type`` is ``None``, it is set to ``"xml"`` for an- :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before- passing it to :class:`parsel.Selector`.+ :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an+ :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to+ :class:`parsel.Selector`, which for any other response is left to+ determine the type from the response body... note:: JSON selector support requires ``parsel`` 1.8.0 or higher. Witholder versions setting ``type`` to ``"json"`` or ``"text"`` is not@@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref):f"{self.__class__.__name__}.__init__() received both response and text")+ # A response that is neither HTML nor XML, e.g. a JSON one, keeps type+ # unset, so that parsel determines it from the body.if type is None:- type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001+ if isinstance(response, XmlResponse):+ type = "xml" # noqa: A001+ elif response is None or isinstance(response, HtmlResponse):+ type = "html" # noqa: A001if text is not None:response = _response_from_text(text, type)diff --git a/tox.ini b/tox.iniindex c56ad011b..94c04f75f 100644--- a/tox.ini+++ b/tox.ini@@ -201,6 +201,51 @@ setenv ={[min]setenv}commands = {[min]commands}+[testenv:vcs-deps]+basepython = python3+deps =+ {[testenv:extra-deps]deps}+ uv+# Dependencies cap each other at their latest release, so their development+# branches usually cannot be resolved together: pyOpenSSL, for one, requires a+# cryptography older than the one cryptography itself is heading towards.+# --no-deps skips resolution entirely, replacing only these distributions and+# leaving the rest of the environment as the install above resolved it.+#+# Pillow and uvloop build from source, and need the libjpeg headers and+# autotools respectively. robotexclusionrulesparser has no public repository,+# so it stays at its latest release.+commands_pre =+ uv pip install --python {envpython} --no-deps --reinstall \+ git+https://github.com/twisted/twisted \+ git+https://github.com/python-pillow/Pillow \+ git+https://github.com/MagicStack/uvloop \+ git+https://github.com/pyca/cryptography \+ git+https://github.com/scrapy/cssselect \+ git+https://github.com/tiran/defusedxml \+ git+https://github.com/scrapy/itemadapter \+ git+https://github.com/scrapy/itemloaders \+ git+https://github.com/lxml/lxml \+ git+https://github.com/pypa/packaging \+ git+https://github.com/scrapy/parsel \+ git+https://github.com/scrapy/protego \+ git+https://github.com/pyca/pyopenssl \+ git+https://github.com/scrapy/queuelib \+ git+https://github.com/pyca/service-identity \+ git+https://github.com/john-kurkowski/tldextract \+ git+https://github.com/scrapy/w3lib \+ git+https://github.com/zopefoundation/zope.interface \+ git+https://github.com/mcfletch/pydispatcher \+ git+https://github.com/boto/boto3 \+ git+https://github.com/bpython/bpython \+ git+https://github.com/google/brotli \+ git+https://github.com/python-hyper/brotlicffi \+ git+https://github.com/googleapis/python-storage \+ git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \+ git+https://github.com/ipython/ipython \+ git+https://github.com/prompt-toolkit/ptpython \+ git+https://github.com/indygreg/python-zstandard+[testenv:default-reactor]commands ={[testenv]commands} --reactor=default