Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Interleave subprocess-heavy tests to avoid an xdist scheduling straggler (#8039)

Adrian4ab7cf3df8ec
diff --git a/conftest.py b/conftest.py
index 35176b672..4c0eca896 100644
--- a/conftest.py
+++ b/conftest.py
@@ -130,7 +130,56 @@ def pytest_configure(config):
install_reactor_import_hook()
+# Test modules made up mostly of tests that spawn a subprocess per test (see
+# tests/utils/cmdline.py). Collected in file order, they form one contiguous
+# block. pytest-xdist's initial scheduling (LoadScheduling.schedule() in
+# xdist/scheduler/load.py) hands each worker one contiguous slice of the
+# collection, so whichever worker's slice overlaps the block ends up running
+# most of it while workers with lighter slices drain them and sit idle for
+# the rest of the run. Spreading these tests evenly through the whole
+# collection instead means every slice gets a proportional share.
+_SUBPROCESS_HEAVY_SUFFIXES = (
+ "tests/test_cmdline/__init__.py",
+ "tests/test_cmdline_crawl_with_pipeline/__init__.py",
+ "tests/test_command_check.py",
+ "tests/test_command_crawl.py",
+ "tests/test_command_fetch.py",
+ "tests/test_command_genspider.py",
+ "tests/test_command_parse.py",
+ "tests/test_command_runspider.py",
+ "tests/test_command_shell.py",
+ "tests/test_command_startproject.py",
+ "tests/test_command_version.py",
+ "tests/test_commands.py",
+ "tests/test_crawler_subprocess.py",
+)
+
+
+def _is_subprocess_heavy(item):
+ return item.path.as_posix().endswith(_SUBPROCESS_HEAVY_SUFFIXES)
+
+
+def _interleave_evenly(items, is_heavy):
+ """Merge *light* and *heavy* items so heavy ones land at roughly even
+ intervals throughout the result, instead of clustered together."""
+ light = [item for item in items if not is_heavy(item)]
+ heavy = [item for item in items if is_heavy(item)]
+ result = []
+ light_i = heavy_i = 0
+ while light_i < len(light) or heavy_i < len(heavy):
+ heavy_progress = heavy_i / len(heavy) if heavy else 1.0
+ light_progress = light_i / len(light) if light else 1.0
+ if heavy_i < len(heavy) and heavy_progress <= light_progress:
+ result.append(heavy[heavy_i])
+ heavy_i += 1
+ else:
+ result.append(light[light_i])
+ light_i += 1
+ return result
+
+
def pytest_collection_modifyitems(items):
+ items[:] = _interleave_evenly(items, _is_subprocess_heavy)
for item in items:
if item.get_closest_marker("requires_internet"):
# Requests to real websites fail every now and then in CI for