Scrapy, a fast high-level web crawling & scraping framework for Python.
Restore sys.path and sys.modules handling in runspider (#8463)
Adrianbae09713cca8
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex d07a789f5..0e9884cc8 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport importlib.utilimport os+import sysfrom importlib.machinery import SourceFileLoaderfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar@@ -30,7 +31,16 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:spec = importlib.util.spec_from_file_location(module_name, abspath, loader=loader)assert specmodule = importlib.util.module_from_spec(spec)- loader.exec_module(module)+ # Registering the module lets code that looks it up by name, e.g.+ # dataclasses and load_object(), find it. A module already registered under+ # that name, e.g. a standard library module, is left alone.+ if module_name not in sys.modules:+ sys.modules[module_name] = module+ sys.path = [str(abspath.parent), *sys.path]+ try:+ loader.exec_module(module)+ finally:+ sys.path.pop(0)return modulediff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.pyindex 4d809c4f3..266b8dc59 100644--- a/tests/test_command_runspider.py+++ b/tests/test_command_runspider.py@@ -176,6 +176,71 @@ class MySpider(scrapy.Spider):assert p.returncode == 0, p.stderrassert "DEBUG: It Works!" in p.stderr+ def test_sibling_import(self, tmp_path: Path) -> None:+ (tmp_path / "spiders").mkdir()+ (tmp_path / "spiders" / "helper_mod.py").write_text(+ "MESSAGE = 'It Works!'\n", encoding="utf-8"+ )+ spider_code = """+import scrapy+from helper_mod import MESSAGE++class MySpider(scrapy.Spider):+ name = 'myspider'++ async def start(self):+ self.logger.debug(MESSAGE)+ return+ yield+"""+ log = self.get_log(+ tmp_path, spider_code, name=f"spiders/{self.spider_filename}"+ )+ assert "DEBUG: It Works!" in log++ def test_dataclass(self, tmp_path: Path) -> None:+ spider_code = """+from __future__ import annotations++from dataclasses import dataclass+from typing import ClassVar++import scrapy++@dataclass+class Item:+ kind: ClassVar[str] = "item"+ message: str = "It Works!"++class MySpider(scrapy.Spider):+ name = 'myspider'++ async def start(self):+ self.logger.debug(Item().message)+ return+ yield+"""+ log = self.get_log(tmp_path, spider_code)+ assert "DEBUG: It Works!" in log++ def test_module_name_taken(self, tmp_path: Path) -> None:+ spider_code = """+import json++import scrapy++class MySpider(scrapy.Spider):+ name = 'myspider'++ async def start(self):+ self.logger.debug(json.loads('"It Works!"'))+ return+ yield+"""+ (tmp_path / "spiders").mkdir()+ log = self.get_log(tmp_path, spider_code, name="spiders/json.py")+ assert "DEBUG: It Works!" in log+def test_start_errors(self, tmp_path: Path) -> None:log = self.get_log(tmp_path, self.badspider, name="badspider.py")assert "start" in log