Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Restore sys.path and sys.modules handling in runspider (#8463)

Adrianbae09713cca8
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index d07a789f5..0e9884cc8 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import importlib.util
import os
+import sys
from importlib.machinery import SourceFileLoader
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar
@@ -30,7 +31,16 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
spec = importlib.util.spec_from_file_location(module_name, abspath, loader=loader)
assert spec
module = importlib.util.module_from_spec(spec)
- loader.exec_module(module)
+ # Registering the module lets code that looks it up by name, e.g.
+ # dataclasses and load_object(), find it. A module already registered under
+ # that name, e.g. a standard library module, is left alone.
+ if module_name not in sys.modules:
+ sys.modules[module_name] = module
+ sys.path = [str(abspath.parent), *sys.path]
+ try:
+ loader.exec_module(module)
+ finally:
+ sys.path.pop(0)
return module
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 4d809c4f3..266b8dc59 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -176,6 +176,71 @@ class MySpider(scrapy.Spider):
assert p.returncode == 0, p.stderr
assert "DEBUG: It Works!" in p.stderr
+ def test_sibling_import(self, tmp_path: Path) -> None:
+ (tmp_path / "spiders").mkdir()
+ (tmp_path / "spiders" / "helper_mod.py").write_text(
+ "MESSAGE = 'It Works!'\n", encoding="utf-8"
+ )
+ spider_code = """
+import scrapy
+from helper_mod import MESSAGE
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ async def start(self):
+ self.logger.debug(MESSAGE)
+ return
+ yield
+"""
+ log = self.get_log(
+ tmp_path, spider_code, name=f"spiders/{self.spider_filename}"
+ )
+ assert "DEBUG: It Works!" in log
+
+ def test_dataclass(self, tmp_path: Path) -> None:
+ spider_code = """
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import ClassVar
+
+import scrapy
+
+@dataclass
+class Item:
+ kind: ClassVar[str] = "item"
+ message: str = "It Works!"
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ async def start(self):
+ self.logger.debug(Item().message)
+ return
+ yield
+"""
+ log = self.get_log(tmp_path, spider_code)
+ assert "DEBUG: It Works!" in log
+
+ def test_module_name_taken(self, tmp_path: Path) -> None:
+ spider_code = """
+import json
+
+import scrapy
+
+class MySpider(scrapy.Spider):
+ name = 'myspider'
+
+ async def start(self):
+ self.logger.debug(json.loads('"It Works!"'))
+ return
+ yield
+"""
+ (tmp_path / "spiders").mkdir()
+ log = self.get_log(tmp_path, spider_code, name="spiders/json.py")
+ assert "DEBUG: It Works!" in log
+
def test_start_errors(self, tmp_path: Path) -> None:
log = self.get_log(tmp_path, self.badspider, name="badspider.py")
assert "start" in log