Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

runspider: support process substitution and other extensionless file paths (#8036)

Adrian5b72ba32d752
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index ebaa60cfe..d07a789f5 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,7 +1,8 @@
from __future__ import annotations
-import sys
-from importlib import import_module
+import importlib.util
+import os
+from importlib.machinery import SourceFileLoader
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar
@@ -17,15 +18,19 @@ if TYPE_CHECKING:
def _import_file(filepath: str | PathLike[str]) -> ModuleType:
- abspath = Path(filepath).resolve()
- if abspath.suffix not in {".py", ".pyw"}:
+ # os.path.abspath(), not Path.resolve(), to avoid following symlinks:
+ # fd paths such as /dev/fd/63 resolve to an unopenable pipe: target.
+ abspath = Path(os.path.abspath(filepath)) # noqa: PTH100
+ if abspath.suffix and abspath.suffix not in {".py", ".pyw"}:
raise ValueError(f"Not a Python source file: {abspath}")
- dirname = str(abspath.parent)
- sys.path = [dirname, *sys.path]
- try:
- module = import_module(abspath.stem)
- finally:
- sys.path.pop(0)
+ module_name = abspath.stem or "spidermodule"
+ # An explicit loader is required: without it extensionless file paths, such
+ # as the /dev/fd paths of process substitution, get no loader assigned.
+ loader = SourceFileLoader(module_name, str(abspath))
+ spec = importlib.util.spec_from_file_location(module_name, abspath, loader=loader)
+ assert spec
+ module = importlib.util.module_from_spec(spec)
+ loader.exec_module(module)
return module
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 113b655ed..46be5cdcf 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -3,11 +3,13 @@ from __future__ import annotations
import asyncio
import inspect
import platform
+import subprocess
import sys
from typing import TYPE_CHECKING
import pytest
+from scrapy.utils.test import get_testenv
from tests.spiders import ExceptionSpider, NoRequestsSpider
from tests.utils.cmdline import proc
@@ -155,6 +157,25 @@ class MySpider(scrapy.Spider):
log = self.get_log(tmp_path, "", name="myspider.txt")
assert "Unable to load" in log
+ @pytest.mark.skipif(
+ platform.system() == "Windows", reason="process substitution requires bash"
+ )
+ def test_runspider_fd(self) -> None:
+ command = (
+ f"{sys.executable} -m scrapy.cmdline runspider"
+ f" <(cat <<'EOF'\n{self.debug_log_spider}\nEOF\n)"
+ )
+ p = subprocess.run(
+ ["bash", "-c", command],
+ check=False,
+ capture_output=True,
+ encoding="utf-8",
+ timeout=15,
+ env=get_testenv(),
+ )
+ assert p.returncode == 0, p.stderr
+ assert "DEBUG: It Works!" in p.stderr
+
def test_start_errors(self, tmp_path: Path) -> None:
log = self.get_log(tmp_path, self.badspider, name="badspider.py")
assert "start" in log