Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.
Python agent-skillsai-scientistbioinformaticschemoinformaticsclaudeclaude-skillsclaudecodeclinical-researchcomputational-biologydata-analysisdrug-discoverygenomicsmaterials-sciencemetabolomicsproteomicsscientific-computingscientific-visualization
"""Tests for the experimental-design generators.
Randomisation and DOE code is easy to get subtly wrong in ways that no error
message reveals -- a block that does not actually balance, an allocation ratio
applied to the wrong arm, a design matrix decoded to the wrong real units. So
the assertions here are about the statistical properties the docstrings
promise, not just about shapes: every permuted block is exactly balanced, a
2:1 ratio produces twice as many treatment units, and a two-level design
decodes to the factor's own low/high values.
Determinism matters as much as correctness: every generator takes a seed, and
an experimental design that cannot be reproduced from its seed is not a design.
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
import pytest
import skill_contract
SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "experimental-design"
SCRIPTS = SKILL_ROOT / "scripts"
sys.path.insert(0, str(SCRIPTS))
np = pytest.importorskip("numpy", reason="experimental-design needs numpy")
pd = pytest.importorskip("pandas", reason="experimental-design needs pandas")
import randomization # noqa: E402
# Both scripts are importable libraries with a worked example under
# `if __name__ == "__main__":` rather than argparse CLIs, so the contract's
# demo-block case applies instead of its `--help` case.
DemoBlockTests = skill_contract.cli.demo_test_case(
SKILL_ROOT, ("doe_designs.py", "randomization.py")
)
def doe():
"""Import the DOE module, skipping when pydoe is absent."""
pytest.importorskip("pydoe", reason="doe_designs needs pydoe")
import doe_designs
return doe_designs
class RatioTests(unittest.TestCase):
def test_no_ratio_means_one_of_each_arm(self) -> None:
self.assertEqual(
randomization._normalize_ratio(["a", "b", "c"], None), ["a", "b", "c"]
)
def test_a_ratio_repeats_each_arm(self) -> None:
self.assertEqual(
randomization._normalize_ratio(["treatment", "control"], (2, 1)),
["treatment", "treatment", "control"],
)
def test_a_mismatched_ratio_length_is_refused(self) -> None:
with self.assertRaisesRegex(ValueError, "one entry per arm"):
randomization._normalize_ratio(["a", "b"], (1, 1, 1))
def test_non_positive_ratio_entries_are_refused(self) -> None:
# A zero-weight arm would silently vanish from the allocation.
for ratio in ((1, 0), (-1, 1), (0, 0)):
with self.subTest(ratio=ratio):
with self.assertRaisesRegex(ValueError, "positive integers"):
randomization._normalize_ratio(["a", "b"], ratio)
def test_fractional_ratios_are_refused_by_every_allocation_method(self) -> None:
# A weight below one used to delete the treatment arm; a weight
# above one silently changed the requested allocation split.
methods = (
(randomization.simple_randomization, 500),
(randomization.block_randomization, 12),
(randomization.stratified_block_randomization, {"siteA": 12}),
(randomization.cluster_randomization, 12),
)
for ratio in ((0.5, 1), (1, 0.5), (1.5, 1), (1, 2.5)):
for method, units in methods:
with self.subTest(ratio=ratio, method=method.__name__):
with self.assertRaisesRegex(ValueError, "positive integers"):
method(units, ratio=ratio, seed=0)
def test_whole_number_floats_preserve_the_requested_block_ratio(self) -> None:
frame = randomization.block_randomization(12, ratio=(2.0, 1.0), seed=0)
for _, rows in frame.groupby("block"):
self.assertEqual(
rows["arm"].value_counts().to_dict(), {"treatment": 4, "control": 2}
)
class SimpleRandomizationTests(unittest.TestCase):
def test_every_unit_is_assigned_exactly_once(self) -> None:
frame = randomization.simple_randomization(25, seed=1)
self.assertEqual(len(frame), 25)
self.assertEqual(list(frame["unit_id"]), list(range(1, 26)))
self.assertTrue(set(frame["arm"]) <= {"treatment", "control"})
def test_the_same_seed_reproduces_the_same_allocation(self) -> None:
first = randomization.simple_randomization(40, seed=7)
second = randomization.simple_randomization(40, seed=7)
pd.testing.assert_frame_equal(first, second)
def test_different_seeds_generally_differ(self) -> None:
first = randomization.simple_randomization(60, seed=1)
second = randomization.simple_randomization(60, seed=2)
self.assertFalse(first["arm"].equals(second["arm"]))
def test_an_allocation_ratio_shifts_the_expected_split(self) -> None:
# Simple randomisation only balances in expectation, so assert the
# direction over a large sample rather than an exact count.
frame = randomization.simple_randomization(
4000, arms=("treatment", "control"), ratio=(3, 1), seed=3
)
share = (frame["arm"] == "treatment").mean()
self.assertAlmostEqual(share, 0.75, delta=0.03)
def test_three_arms_are_supported(self) -> None:
frame = randomization.simple_randomization(
300, arms=("a", "b", "c"), seed=5
)
self.assertEqual(set(frame["arm"]), {"a", "b", "c"})
class BlockRandomizationTests(unittest.TestCase):
def test_each_complete_block_is_exactly_balanced(self) -> None:
# This is the entire point of permuted blocks: the requested ratio holds at
# completed-block boundaries, not necessarily at interim prefixes.
frame = randomization.block_randomization(24, block_size=4, seed=1)
for block, rows in frame.groupby("block"):
with self.subTest(block=block):
counts = rows["arm"].value_counts().to_dict()
self.assertEqual(counts, {"treatment": 2, "control": 2})
def test_the_allocation_ratio_holds_within_every_block(self) -> None:
frame = randomization.block_randomization(
18, arms=["treatment", "control"], ratio=(2, 1), block_size=6, seed=1
)
for block, rows in frame.groupby("block"):
with self.subTest(block=block):
counts = rows["arm"].value_counts().to_dict()
self.assertEqual(counts, {"treatment": 4, "control": 2})
def test_a_block_size_incompatible_with_the_ratio_is_refused(self) -> None:
with self.assertRaisesRegex(ValueError, "must be a multiple of"):
randomization.block_randomization(12, ratio=(2, 1), block_size=4)
def test_the_default_block_size_is_valid_for_the_ratio(self) -> None:
frame = randomization.block_randomization(12, ratio=(2, 1), seed=1)
sizes = frame.groupby("block").size().unique()
self.assertEqual(list(sizes), [6])
def test_a_partial_final_block_is_truncated_not_padded(self) -> None:
frame = randomization.block_randomization(10, block_size=4, seed=1)
self.assertEqual(len(frame), 10)
self.assertEqual(list(frame["unit_id"]), list(range(1, 11)))
def test_the_same_seed_reproduces_the_same_blocks(self) -> None:
pd.testing.assert_frame_equal(
randomization.block_randomization(20, seed=11),
randomization.block_randomization(20, seed=11),
)
class StratifiedTests(unittest.TestCase):
def test_interleaved_strata_keep_each_original_units_label(self) -> None:
labels = ["south", "north", "south", "north", "north", "south", "north", "south"]
frame = randomization.stratified_block_randomization(labels, block_size=4, seed=8)
self.assertEqual(list(frame["unit_id"]), list(range(1, 9)))
self.assertEqual(list(frame["stratum"]), labels)
for _, rows in frame.groupby("stratum"):
self.assertEqual(rows["arm"].value_counts().to_dict(),
{"treatment": 2, "control": 2})
def test_every_stratum_is_balanced_independently(self) -> None:
frame = randomization.stratified_block_randomization(
{"siteA": 8, "siteB": 12}, block_size=4, seed=1
)
self.assertEqual(len(frame), 20)
balance = randomization.arm_balance(frame, by="stratum")
self.assertEqual(balance.loc["siteA", "treatment"], 4)
self.assertEqual(balance.loc["siteA", "control"], 4)
self.assertEqual(balance.loc["siteB", "treatment"], 6)
self.assertEqual(balance.loc["siteB", "control"], 6)
def test_unit_ids_are_renumbered_across_the_whole_cohort(self) -> None:
frame = randomization.stratified_block_randomization(
{"a": 4, "b": 4}, block_size=4, seed=1
)
self.assertEqual(list(frame["unit_id"]), list(range(1, 9)))
def test_a_label_sequence_is_accepted_as_well_as_a_count_map(self) -> None:
labels = ["north"] * 4 + ["south"] * 8
frame = randomization.stratified_block_randomization(
labels, block_size=4, seed=1
)
self.assertEqual(len(frame), 12)
self.assertEqual(
frame["stratum"].value_counts().to_dict(), {"south": 8, "north": 4}
)
def test_strata_are_seeded_apart_so_they_do_not_share_a_pattern(self) -> None:
# Every stratum getting the same permutation would defeat stratification.
frame = randomization.stratified_block_randomization(
{"a": 8, "b": 8}, block_size=4, seed=1
)
first = list(frame[frame["stratum"] == "a"]["arm"])
second = list(frame[frame["stratum"] == "b"]["arm"])
self.assertNotEqual(first, second)
class ClusterTests(unittest.TestCase):
def test_one_row_per_cluster_keyed_by_cluster_id(self) -> None:
frame = randomization.cluster_randomization(
["clinic-1", "clinic-2", "clinic-3", "clinic-4"], seed=1
)
self.assertEqual(len(frame), 4)
self.assertEqual(
list(frame["cluster_id"]),
["clinic-1", "clinic-2", "clinic-3", "clinic-4"],
)
self.assertNotIn("unit_id", frame.columns)
def test_an_integer_count_generates_cluster_labels(self) -> None:
frame = randomization.cluster_randomization(3, seed=1)
self.assertEqual(
list(frame["cluster_id"]), ["cluster_1", "cluster_2", "cluster_3"]
)
def test_clusters_are_blocked_so_arms_stay_balanced(self) -> None:
frame = randomization.cluster_randomization(8, block_size=4, seed=1)
counts = frame["arm"].value_counts().to_dict()
self.assertEqual(counts, {"treatment": 4, "control": 4})
class RunOrderTests(unittest.TestCase):
def test_run_order_is_a_permutation_of_every_row(self) -> None:
design = pd.DataFrame({"temp": [20, 20, 60, 60], "ph": [6, 8, 6, 8]})
randomized = randomization.assign_factorial_runs(design, seed=1)
self.assertEqual(sorted(randomized["run_order"]), [1, 2, 3, 4])
self.assertEqual(len(randomized), len(design))
def test_rows_are_returned_sorted_by_run_order(self) -> None:
design = pd.DataFrame({"x": range(10)})
randomized = randomization.assign_factorial_runs(design, seed=2)
self.assertEqual(list(randomized["run_order"]), list(range(1, 11)))
def test_the_original_design_is_not_mutated(self) -> None:
design = pd.DataFrame({"x": range(5)})
before = design.copy()
randomization.assign_factorial_runs(design, seed=1)
pd.testing.assert_frame_equal(design, before)
def test_randomising_actually_reorders_the_runs(self) -> None:
# A systematic order confounds factors with drift, which is the whole
# reason this function exists.
design = pd.DataFrame({"x": range(12)})
randomized = randomization.assign_factorial_runs(design, seed=1)
self.assertNotEqual(list(randomized["x"]), list(range(12)))
class BalanceReportTests(unittest.TestCase):
def test_counts_are_reported_per_arm(self) -> None:
frame = pd.DataFrame({"arm": ["a", "a", "b"]})
self.assertEqual(randomization.arm_balance(frame).to_dict(), {"a": 2, "b": 1})
def test_grouping_produces_a_stratum_by_arm_table(self) -> None:
frame = pd.DataFrame(
{"arm": ["a", "b", "a", "a"], "stratum": ["x", "x", "y", "y"]}
)
table = randomization.arm_balance(frame, by="stratum")
self.assertEqual(table.loc["x", "a"], 1)
self.assertEqual(table.loc["y", "a"], 2)
# unstack(fill_value=0) -- an absent combination is 0, not NaN.
self.assertEqual(table.loc["y", "b"], 0)
class DesignMatrixTests(unittest.TestCase):
def test_fraction_has_expected_resolution_four_aliases(self) -> None:
factors = {name: (-1, 1) for name in "ABCD"}
design = doe().fractional_factorial(factors, "a b c abc", randomize=False)
np.testing.assert_array_equal(design["A"] * design["B"],
design["C"] * design["D"])
np.testing.assert_allclose(design.to_numpy().T @ design.to_numpy(),
8 * np.eye(4))
def test_central_composite_geometry_and_quadratic_rank(self) -> None:
factors = {"x": (-1, 1), "y": (-1, 1)}
for face in ("circumscribed", "inscribed", "faced"):
with self.subTest(face=face):
design = doe().central_composite(factors, center=(2, 2), face=face,
randomize=False)
x, y = design["x"].to_numpy(), design["y"].to_numpy()
model = np.column_stack((np.ones(len(x)), x, y, x*y, x*x, y*y))
self.assertEqual(np.linalg.matrix_rank(model), 6)
self.assertEqual(int(((x == 0) & (y == 0)).sum()), 4)
if face == "circumscribed":
self.assertGreater(np.abs(design.to_numpy()).max(), 1)
else:
self.assertLessEqual(np.abs(design.to_numpy()).max(), 1)
def test_latin_hypercube_uses_each_interval_exactly_once(self) -> None:
for criterion in (None, "center", "maximin", "centermaximin", "correlation"):
with self.subTest(criterion=criterion):
design = doe().latin_hypercube({"x": (0, 1), "y": (0, 1)}, 12,
criterion=criterion, seed=8)
for column in design:
self.assertEqual(sorted(np.floor(12 * design[column]).astype(int)),
list(range(12)))
def test_full_factorial_covers_every_combination(self) -> None:
design = doe().full_factorial(
{"temp": [20, 40, 60], "catalyst": ["A", "B"]}, randomize=False
)
self.assertEqual(len(design), 6)
combinations = set(zip(design["temp"], design["catalyst"]))
self.assertEqual(len(combinations), 6)
def test_two_level_factorial_decodes_to_the_stated_low_and_high(self) -> None:
design = doe().two_level_factorial(
{"temp": (20, 60), "ph": (6.0, 8.0)}, randomize=False
)
self.assertEqual(len(design), 4)
self.assertEqual(set(design["temp"]), {20.0, 60.0})
self.assertEqual(set(design["ph"]), {6.0, 8.0})
def test_a_generator_that_names_the_wrong_factor_count_is_refused(self) -> None:
with self.assertRaisesRegex(ValueError, "generator defines"):
doe().fractional_factorial({"a": (0, 1), "b": (0, 1)}, "a b c")
def test_a_valid_fraction_halves_the_full_factorial(self) -> None:
factors = {f: (0, 1) for f in ("a", "b", "c", "d")}
design = doe().fractional_factorial(factors, "a b c abc", randomize=False)
self.assertEqual(len(design), 8) # 2^(4-1), not 2^4
self.assertEqual(list(design.columns), list(factors))
def test_plackett_burman_drops_the_dummy_columns(self) -> None:
factors = {f: (0, 1) for f in ("a", "b", "c", "d", "e")}
design = doe().plackett_burman(factors, randomize=False)
self.assertEqual(list(design.columns), list(factors))
def test_box_behnken_needs_three_factors(self) -> None:
with self.assertRaisesRegex(ValueError, "at least 3 factors"):
doe().box_behnken({"a": (0, 1), "b": (0, 1)})
def test_box_behnken_avoids_the_extreme_corners(self) -> None:
# Never using the all-low / all-high corner is the design's reason for
# existing -- those runs are the unsafe or infeasible ones.
factors = {"a": (0, 10), "b": (0, 10), "c": (0, 10)}
design = doe().box_behnken(factors, randomize=False)
corners = {(0.0, 0.0, 0.0), (10.0, 10.0, 10.0)}
rows = set(zip(design["a"], design["b"], design["c"]))
self.assertFalse(rows & corners)
def test_latin_hypercube_stays_inside_every_factor_range(self) -> None:
design = doe().latin_hypercube({"x": (2.0, 5.0), "y": (-1.0, 1.0)}, 20, seed=3)
self.assertEqual(len(design), 20)
self.assertTrue((design["x"] >= 2.0).all() and (design["x"] <= 5.0).all())
self.assertTrue((design["y"] >= -1.0).all() and (design["y"] <= 1.0).all())
def test_latin_hypercube_is_reproducible_from_its_seed(self) -> None:
factors = {"x": (0.0, 1.0), "y": (0.0, 1.0)}
pd.testing.assert_frame_equal(
doe().latin_hypercube(factors, 15, seed=9),
doe().latin_hypercube(factors, 15, seed=9),
)
def test_randomize_adds_a_run_order_column_and_sorts_by_it(self) -> None:
factors = {"a": (0, 1), "b": (0, 1)}
design = doe().two_level_factorial(factors, randomize=True, seed=4)
self.assertEqual(design.columns[0], "run_order")
self.assertEqual(list(design["run_order"]), [1, 2, 3, 4])
def test_not_randomizing_leaves_the_design_in_canonical_order(self) -> None:
factors = {"a": (0, 1), "b": (0, 1)}
design = doe().two_level_factorial(factors, randomize=False)
self.assertNotIn("run_order", design.columns)
@pytest.mark.parametrize("n", [-1, 2.5, True])
def test_invalid_allocation_counts_are_rejected(n):
for method in (randomization.simple_randomization, randomization.block_randomization):
with pytest.raises(ValueError, match="n must be an integer"):
method(n)
@pytest.mark.parametrize("block_size", [0, -4, 4.5, True])
def test_nonpositive_or_fractional_blocks_cannot_hang(block_size):
with pytest.raises(ValueError, match="block_size must be an integer"):
randomization.block_randomization(8, block_size=block_size)
@pytest.mark.parametrize("arms", [[], ["a"], ["a", "a"], ["a", None]])
def test_arms_must_identify_distinct_nonmissing_groups(arms):
with pytest.raises(ValueError, match="arms must contain"):
randomization.block_randomization(8, arms=arms)
def test_missing_stratum_cannot_silently_drop_a_subject():
with pytest.raises(ValueError, match="strata must contain"):
randomization.stratified_block_randomization(["a", None, "b"])
def test_duplicate_cluster_ids_cannot_assign_one_cluster_twice():
with pytest.raises(ValueError, match="clusters must contain"):
randomization.cluster_randomization(["clinic1", "clinic1"])
def test_zero_subjects_produce_empty_schedules():
assert randomization.simple_randomization(0).empty
assert randomization.block_randomization(0).empty
for strata in ([], {}, {"a": 0}):
frame = randomization.stratified_block_randomization(strata)
assert frame.empty
assert list(frame) == ["unit_id", "stratum", "block", "arm"]
@pytest.mark.parametrize("levels", [(1, 1), (5, 2), (1, 2, 3), (0, np.inf), (np.nan, 1)])
def test_invalid_ranges_cannot_silently_change_the_design(levels):
with pytest.raises(ValueError, match="exactly two finite numeric levels"):
doe().two_level_factorial({"x": levels})
def test_plackett_burman_current_supported_and_unsupported_run_counts():
design = doe().plackett_burman({f"x{i}": (-1, 1) for i in range(11)}, randomize=False)
assert design.shape == (12, 11)
np.testing.assert_allclose(design.to_numpy().T @ design.to_numpy(), 12 * np.eye(11))
with pytest.raises(ValueError):
doe().plackett_burman({f"x{i}": (-1, 1) for i in range(25)}, randomize=False)
if __name__ == "__main__":
unittest.main()