Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add VCS CI job, fix support for upcoming parsel version (#7924)

Adrian94dff69468a9
diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml
new file mode 100644
index 000000000..bf867dba7
--- /dev/null
+++ b/.github/workflows/tests-vcs-deps.yml
@@ -0,0 +1,53 @@
+name: VCS dependencies
+
+permissions:
+ contents: read
+
+on:
+ schedule:
+ - cron: '0 4 * * *'
+ workflow_dispatch:
+
+concurrency:
+ group: ${{github.workflow}}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ tests:
+ name: tests
+ runs-on: ubuntu-latest
+ env:
+ PYTEST_ADDOPTS: -n auto --no-cov
+ TOXENV: vcs-deps
+ UV_PYTHON_PREFERENCE: only-system
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.14"
+
+ # Dependencies that ship wheels on PyPI are built from source here, so
+ # their build dependencies are needed: libxml2 and libxslt for lxml,
+ # libjpeg and zlib for Pillow, and autotools for the libuv bundled in
+ # uvloop.
+ - name: Install system libraries
+ run: |
+ sudo apt-get update
+ sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev
+
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ run: uv tool install --python cpython mitmproxy
+
+ - name: Run tests
+ run: uvx --with tox-uv tox
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index f6334c32c..fa91e2904 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref):
``"json"``, ``"text"`` or ``None`` (default). It's passed to
:class:`parsel.Selector` and its meaning is defined there. However, when
``type`` is ``None``, it is set to ``"xml"`` for an
- :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
- passing it to :class:`parsel.Selector`.
+ :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an
+ :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to
+ :class:`parsel.Selector`, which for any other response is left to
+ determine the type from the response body.
.. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
older versions setting ``type`` to ``"json"`` or ``"text"`` is not
@@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref):
f"{self.__class__.__name__}.__init__() received both response and text"
)
+ # A response that is neither HTML nor XML, e.g. a JSON one, keeps type
+ # unset, so that parsel determines it from the body.
if type is None:
- type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001
+ if isinstance(response, XmlResponse):
+ type = "xml" # noqa: A001
+ elif response is None or isinstance(response, HtmlResponse):
+ type = "html" # noqa: A001
if text is not None:
response = _response_from_text(text, type)
diff --git a/tox.ini b/tox.ini
index c56ad011b..94c04f75f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -201,6 +201,51 @@ setenv =
{[min]setenv}
commands = {[min]commands}
+[testenv:vcs-deps]
+basepython = python3
+deps =
+ {[testenv:extra-deps]deps}
+ uv
+# Dependencies cap each other at their latest release, so their development
+# branches usually cannot be resolved together: pyOpenSSL, for one, requires a
+# cryptography older than the one cryptography itself is heading towards.
+# --no-deps skips resolution entirely, replacing only these distributions and
+# leaving the rest of the environment as the install above resolved it.
+#
+# Pillow and uvloop build from source, and need the libjpeg headers and
+# autotools respectively. robotexclusionrulesparser has no public repository,
+# so it stays at its latest release.
+commands_pre =
+ uv pip install --python {envpython} --no-deps --reinstall \
+ git+https://github.com/twisted/twisted \
+ git+https://github.com/python-pillow/Pillow \
+ git+https://github.com/MagicStack/uvloop \
+ git+https://github.com/pyca/cryptography \
+ git+https://github.com/scrapy/cssselect \
+ git+https://github.com/tiran/defusedxml \
+ git+https://github.com/scrapy/itemadapter \
+ git+https://github.com/scrapy/itemloaders \
+ git+https://github.com/lxml/lxml \
+ git+https://github.com/pypa/packaging \
+ git+https://github.com/scrapy/parsel \
+ git+https://github.com/scrapy/protego \
+ git+https://github.com/pyca/pyopenssl \
+ git+https://github.com/scrapy/queuelib \
+ git+https://github.com/pyca/service-identity \
+ git+https://github.com/john-kurkowski/tldextract \
+ git+https://github.com/scrapy/w3lib \
+ git+https://github.com/zopefoundation/zope.interface \
+ git+https://github.com/mcfletch/pydispatcher \
+ git+https://github.com/boto/boto3 \
+ git+https://github.com/bpython/bpython \
+ git+https://github.com/google/brotli \
+ git+https://github.com/python-hyper/brotlicffi \
+ git+https://github.com/googleapis/python-storage \
+ git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \
+ git+https://github.com/ipython/ipython \
+ git+https://github.com/prompt-toolkit/ptpython \
+ git+https://github.com/indygreg/python-zstandard
+
[testenv:default-reactor]
commands =
{[testenv]commands} --reactor=default