Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

2.1 KB
Raw Blame
import ast
import json
import os
import pstats
import shutil
import sys
import tempfile
from io import StringIO
from pathlib import Path
from subprocess import PIPE, Popen
from typing import Any

from scrapy.utils.test import get_testenv


def _execute(*new_args: str, **kwargs: Any) -> str:
    env = get_testenv()
    tests_path = Path(__file__).parent.parent
    env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
    env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
    encoding = sys.stdout.encoding or "utf-8"
    args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
    proc: Popen[bytes] = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
    comm = proc.communicate()[0].strip()
    return comm.decode(encoding)


def test_default_settings():
    assert _execute("settings", "--get", "TEST1") == "default"


def test_override_settings_using_set_arg():
    assert _execute("settings", "--get", "TEST1", "-s", "TEST1=override") == "override"


def test_profiling():
    path = Path(tempfile.mkdtemp())
    filename = path / "res.prof"
    try:
        _execute("version", "--profile", str(filename))
        assert filename.exists()
        out = StringIO()
        stats = pstats.Stats(str(filename), stream=out)
        stats.print_stats()
        out.seek(0)
        stats_text = out.read()
        assert str(Path("scrapy", "commands", "version.py")) in stats_text
        assert "tottime" in stats_text
    finally:
        shutil.rmtree(path)


def test_override_dict_settings():
    EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
    EXTENSIONS = {EXT_PATH: 200}
    settingsstr = _execute(
        "settings",
        "--get",
        "EXTENSIONS",
        "-s",
        "EXTENSIONS=" + json.dumps(EXTENSIONS),
    )
    assert "..." not in settingsstr
    settingsdict = ast.literal_eval(settingsstr)
    assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
    assert settingsdict[EXT_PATH] == 200


def test_pathlib_path_as_feeds_key():
    assert _execute("settings", "--get", "FEEDS") == json.dumps(
        {"items.csv": {"format": "csv", "fields": ["price", "name"]}}
    )