Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add the project directory to sys.path even when SCRAPY_SETTINGS_MODULE is set (#8042)

Adrianca1f32612cb6
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 4850b370b..3b23d2d6c 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -92,7 +92,9 @@ def init_env(project: str = "default", set_syspath: bool = True) -> None:
be able to locate the project module.
"""
cfg = get_config()
- if cfg.has_option("settings", project):
+ if "SCRAPY_SETTINGS_MODULE" not in os.environ and cfg.has_option(
+ "settings", project
+ ):
os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project)
closest = closest_scrapy_cfg()
if closest:
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 406d6d960..808017ee0 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -116,9 +116,7 @@ def get_project_settings() -> Settings:
Settings from sources with a higher precedence, such as :ref:`spider
settings <spider-settings>`, are applied when a crawl starts.
"""
- if ENVVAR not in os.environ:
- project = os.environ.get("SCRAPY_PROJECT", "default")
- init_env(project)
+ init_env(os.environ.get("SCRAPY_PROJECT", "default"))
settings = Settings()
settings_module_path = os.environ.get(ENVVAR)
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 297ae4414..be64c8b36 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import os
+import sys
from pathlib import Path
from typing import TYPE_CHECKING
@@ -124,6 +125,14 @@ class TestGetProjectSettings:
assert settings.get("SCRAPY_FOO") is None
+ def test_envvar_project_dir_in_sys_path(
+ self, proj_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(sys, "path", sys.path.copy())
+ monkeypatch.setenv("SCRAPY_SETTINGS_MODULE", "tests.test_cmdline.settings")
+ get_project_settings()
+ assert str(proj_path) in sys.path
+
def test_valid_and_invalid_envvars(self):
value = "tests.test_cmdline.settings"
envvars = {