Scrapy, a fast high-level web crawling & scraping framework for Python.
runspider: support process substitution and other extensionless file paths (#8036)
Adrian5b72ba32d752
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex ebaa60cfe..d07a789f5 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -1,7 +1,8 @@from __future__ import annotations-import sys-from importlib import import_module+import importlib.util+import os+from importlib.machinery import SourceFileLoaderfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar@@ -17,15 +18,19 @@ if TYPE_CHECKING:def _import_file(filepath: str | PathLike[str]) -> ModuleType:- abspath = Path(filepath).resolve()- if abspath.suffix not in {".py", ".pyw"}:+ # os.path.abspath(), not Path.resolve(), to avoid following symlinks:+ # fd paths such as /dev/fd/63 resolve to an unopenable pipe: target.+ abspath = Path(os.path.abspath(filepath)) # noqa: PTH100+ if abspath.suffix and abspath.suffix not in {".py", ".pyw"}:raise ValueError(f"Not a Python source file: {abspath}")- dirname = str(abspath.parent)- sys.path = [dirname, *sys.path]- try:- module = import_module(abspath.stem)- finally:- sys.path.pop(0)+ module_name = abspath.stem or "spidermodule"+ # An explicit loader is required: without it extensionless file paths, such+ # as the /dev/fd paths of process substitution, get no loader assigned.+ loader = SourceFileLoader(module_name, str(abspath))+ spec = importlib.util.spec_from_file_location(module_name, abspath, loader=loader)+ assert spec+ module = importlib.util.module_from_spec(spec)+ loader.exec_module(module)return modulediff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 113b655ed..46be5cdcf 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -3,11 +3,13 @@ from __future__ import annotationsimport asyncioimport inspectimport platform+import subprocessimport sysfrom typing import TYPE_CHECKINGimport pytest+from scrapy.utils.test import get_testenvfrom tests.spiders import ExceptionSpider, NoRequestsSpiderfrom tests.utils.cmdline import proc@@ -155,6 +157,25 @@ class MySpider(scrapy.Spider):log = self.get_log(tmp_path, "", name="myspider.txt")assert "Unable to load" in log+ @pytest.mark.skipif(+ platform.system() == "Windows", reason="process substitution requires bash"+ )+ def test_runspider_fd(self) -> None:+ command = (+ f"{sys.executable} -m scrapy.cmdline runspider"+ f" <(cat <<'EOF'\n{self.debug_log_spider}\nEOF\n)"+ )+ p = subprocess.run(+ ["bash", "-c", command],+ check=False,+ capture_output=True,+ encoding="utf-8",+ timeout=15,+ env=get_testenv(),+ )+ assert p.returncode == 0, p.stderr+ assert "DEBUG: It Works!" in p.stderr+def test_start_errors(self, tmp_path: Path) -> None:log = self.get_log(tmp_path, self.badspider, name="badspider.py")assert "start" in log