Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.
Python agent-skillsai-scientistbioinformaticschemoinformaticsclaudeclaude-skillsclaudecodeclinical-researchcomputational-biologydata-analysisdrug-discoverygenomicsmaterials-sciencemetabolomicsproteomicsscientific-computingscientific-visualization
"""Tests for the PyTorch Lightning module, datamodule, and trainer templates.
A template is only useful if it runs, and these had three ways of not running.
`configure_optimizers` passed `verbose=True` to `ReduceLROnPlateau`, which
PyTorch removed, so every model built from the template raised TypeError before
the first step. Two trainer builders asked `ModelCheckpoint` for the top 3
checkpoints without a `monitor`, which Lightning rejects outright -- so those
functions could not be called on any machine. And the module demo printed
`model.num_parameters`, an attribute LightningModule does not have.
Beyond that the tests assert the configuration the templates claim, because
claiming it is the whole point: Adam at the requested learning rate with 1e-5
weight decay, a plateau scheduler watching `val/loss` on an epoch interval, a
784 -> hidden -> 10 network whose parameter count is arithmetic, and a
train/val split that is reproducible because it is seeded. One short CPU fit
then proves the pieces compose -- in particular that the metric the scheduler
monitors is one the module actually logs, which Lightning enforces at runtime.
CUDA-only builders skip when CUDA hardware is absent. The companion contract
suite runs the production callback configuration on CPU with explicit overrides;
GPU/FSDP/DeepSpeed execution remains unvalidated here.
"""
from __future__ import annotations
import sys
import tempfile
import unittest
from pathlib import Path
import pytest
import skill_contract
SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "pytorch-lightning"
SCRIPTS = SKILL_ROOT / "scripts"
sys.path.insert(0, str(SCRIPTS))
torch = pytest.importorskip("torch", reason="the templates need torch")
L = pytest.importorskip("lightning", reason="the templates need lightning")
from lightning.pytorch.accelerators import CUDAAccelerator, MPSAccelerator # noqa: E402
from lightning.pytorch.callbacks import ( # noqa: E402
EarlyStopping,
LearningRateMonitor,
ModelCheckpoint,
)
from torch.utils.data import DataLoader, TensorDataset # noqa: E402
import quick_trainer_setup # noqa: E402
import template_datamodule # noqa: E402
import template_lightning_module # noqa: E402
# All three scripts are importable templates with a worked example under
# `if __name__ == "__main__"`, and none builds an argparse parser.
DemoBlockTests = skill_contract.cli.demo_test_case(
SKILL_ROOT,
(
"template_lightning_module.py",
"template_datamodule.py",
"quick_trainer_setup.py",
),
)
CUDA_DEVICES = CUDAAccelerator.auto_device_count() if CUDAAccelerator.is_available() else 0
def build_or_skip(test: unittest.TestCase, builder, *args, **kwargs):
"""Only unavailable CUDA hardware skips; construction errors must fail."""
if not CUDA_DEVICES:
test.skipTest("CUDA hardware unavailable")
return builder(*args, **kwargs)
class ModuleArchitectureTests(unittest.TestCase):
def test_the_hyperparameters_are_saved_and_reachable(self) -> None:
# save_hyperparameters() is what makes checkpoints reloadable without
# repeating the constructor arguments.
module = template_lightning_module.TemplateLightningModule(
learning_rate=0.01, hidden_dim=32, dropout=0.25
)
self.assertEqual(
dict(module.hparams),
{"learning_rate": 0.01, "hidden_dim": 32, "dropout": 0.25},
)
def test_the_network_maps_784_inputs_to_10_logits(self) -> None:
module = template_lightning_module.TemplateLightningModule(hidden_dim=16)
output = module(torch.randn(5, 784))
self.assertEqual(tuple(output.shape), (5, 10))
def test_the_parameter_count_is_the_two_dense_layers(self) -> None:
# 784*h + h + h*10 + 10, which the demo block now prints correctly.
for hidden_dim in (8, 256):
with self.subTest(hidden_dim=hidden_dim):
module = template_lightning_module.TemplateLightningModule(
hidden_dim=hidden_dim
)
self.assertEqual(
sum(p.numel() for p in module.parameters()),
784 * hidden_dim + hidden_dim + hidden_dim * 10 + 10,
)
def test_dropout_is_active_in_training_and_disabled_in_eval(self) -> None:
# A template that leaves dropout on at inference reports noisy metrics.
module = template_lightning_module.TemplateLightningModule(
hidden_dim=64, dropout=0.9
)
batch = torch.randn(8, 784)
module.train()
torch.manual_seed(0)
first = module(batch)
torch.manual_seed(1)
second = module(batch)
self.assertFalse(torch.allclose(first, second))
module.eval()
with torch.no_grad():
self.assertTrue(torch.allclose(module(batch), module(batch)))
class OptimiserConfigurationTests(unittest.TestCase):
def setUp(self) -> None:
self.module = template_lightning_module.TemplateLightningModule(
learning_rate=0.003, hidden_dim=8
)
# The regression: ReduceLROnPlateau dropped `verbose`, so this call
# raised TypeError and no model built from the template could train.
self.configuration = self.module.configure_optimizers()
def test_it_returns_an_adam_optimiser_at_the_requested_learning_rate(self) -> None:
optimizer = self.configuration["optimizer"]
self.assertIsInstance(optimizer, torch.optim.Adam)
self.assertEqual(optimizer.param_groups[0]["lr"], 0.003)
self.assertEqual(optimizer.param_groups[0]["weight_decay"], 1e-5)
def test_the_optimiser_owns_every_model_parameter(self) -> None:
optimised = sum(
p.numel() for group in self.configuration["optimizer"].param_groups
for p in group["params"]
)
self.assertEqual(optimised, sum(p.numel() for p in self.module.parameters()))
def test_the_scheduler_halves_the_rate_on_a_five_epoch_plateau(self) -> None:
scheduler = self.configuration["lr_scheduler"]["scheduler"]
self.assertIsInstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau)
self.assertEqual(scheduler.mode, "min")
self.assertEqual(scheduler.factor, 0.5)
self.assertEqual(scheduler.patience, 5)
def test_the_scheduler_watches_a_metric_the_module_logs(self) -> None:
# Lightning raises at the end of the first epoch if the monitored key
# was never logged, so these two have to agree.
self.assertEqual(self.configuration["lr_scheduler"]["monitor"], "val/loss")
self.assertEqual(self.configuration["lr_scheduler"]["interval"], "epoch")
self.assertEqual(self.configuration["lr_scheduler"]["frequency"], 1)
def test_the_plateau_scheduler_actually_reduces_the_rate(self) -> None:
# patience=5 means six non-improving epochs before the drop.
scheduler = self.configuration["lr_scheduler"]["scheduler"]
optimizer = self.configuration["optimizer"]
for _ in range(7):
scheduler.step(1.0)
self.assertAlmostEqual(optimizer.param_groups[0]["lr"], 0.0015)
class DataModuleTests(unittest.TestCase):
"""The synthetic vectors match the model and use isolated evaluation views."""
@classmethod
def setUpClass(cls) -> None:
cls.datamodule = template_datamodule.TemplateDataModule(
data_dir="./data", batch_size=8, num_workers=0, train_val_split=0.75
)
cls.datamodule.prepare_data()
cls.datamodule.setup(stage="fit")
def test_the_split_honours_the_requested_ratio(self) -> None:
self.assertEqual(len(self.datamodule.train_dataset), 750)
self.assertEqual(len(self.datamodule.val_dataset), 250)
self.assertEqual(
len(self.datamodule.train_dataset) + len(self.datamodule.val_dataset), 1000
)
def test_the_split_is_seeded_so_two_runs_agree(self) -> None:
# random_split is given an explicit generator seed; without it the
# validation set would change every run and every resumed checkpoint.
other = template_datamodule.TemplateDataModule(
data_dir="./data", batch_size=8, num_workers=0, train_val_split=0.75
)
other.setup(stage="fit")
self.assertEqual(
list(other.train_dataset.indices), list(self.datamodule.train_dataset.indices)
)
self.assertEqual(
set(other.train_dataset.indices) & set(other.val_dataset.indices), set()
)
def test_the_training_loader_shuffles_and_keeps_the_short_batch(self) -> None:
loader = self.datamodule.train_dataloader()
self.assertFalse(loader.drop_last)
self.assertIsInstance(loader.sampler, torch.utils.data.RandomSampler)
# Keep the final six examples.
self.assertEqual(len(loader), 94)
def test_the_validation_loader_keeps_order_and_every_sample(self) -> None:
loader = self.datamodule.val_dataloader()
self.assertIsInstance(loader.sampler, torch.utils.data.SequentialSampler)
self.assertFalse(loader.drop_last)
# ceil(250 / 8) = 32 batches, so nothing is silently discarded.
self.assertEqual(len(loader), 32)
def test_a_batch_has_the_declared_shape_and_dtypes(self) -> None:
samples, labels = next(iter(self.datamodule.train_dataloader()))
self.assertEqual(tuple(samples.shape), (8, 784))
self.assertEqual(tuple(labels.shape), (8,))
self.assertEqual(samples.dtype, torch.float32)
self.assertEqual(labels.dtype, torch.int64)
self.assertTrue(bool(((labels >= 0) & (labels < 10)).all()))
def test_a_different_ratio_moves_the_boundary(self) -> None:
datamodule = template_datamodule.TemplateDataModule(
data_dir="./data", batch_size=32, num_workers=0, train_val_split=0.6
)
datamodule.setup(stage="fit")
self.assertEqual(len(datamodule.train_dataset), 600)
self.assertEqual(len(datamodule.val_dataset), 400)
def test_the_test_and_predict_stages_build_their_own_datasets(self) -> None:
datamodule = template_datamodule.TemplateDataModule(
data_dir="./data", batch_size=64, num_workers=0
)
datamodule.setup(stage="test")
self.assertEqual(len(datamodule.test_dataset), 1000)
self.assertIsNone(datamodule.predict_dataset)
datamodule.setup(stage="predict")
self.assertEqual(len(datamodule.predict_dataset), 1000)
self.assertEqual(len(datamodule.test_dataloader()), 16)
def test_the_checkpointed_state_round_trips(self) -> None:
datamodule = template_datamodule.TemplateDataModule(train_val_split=0.8)
self.assertEqual(datamodule.state_dict(), {"train_val_split": 0.8, "seed": 42, "num_samples": 1000})
datamodule.load_state_dict({"train_val_split": 0.5})
self.assertEqual(datamodule.hparams.train_val_split, 0.5)
self.assertEqual(datamodule.state_dict(), {"train_val_split": 0.5, "seed": 42, "num_samples": 1000})
def test_teardown_releases_only_the_stage_it_was_given(self) -> None:
datamodule = template_datamodule.TemplateDataModule(num_workers=0)
datamodule.setup(stage="test")
datamodule.teardown(stage="predict")
self.assertIsNotNone(datamodule.test_dataset)
datamodule.teardown(stage="test")
self.assertIsNone(datamodule.test_dataset)
def test_a_transform_is_applied_to_every_sample(self) -> None:
dataset = template_datamodule.CustomDataset(
"./data", transform=lambda sample: torch.zeros_like(sample)
)
sample, label = dataset[0]
self.assertTrue(bool((sample == 0).all()))
# The label is never passed through the transform.
self.assertEqual(label.dtype, torch.int64)
self.assertEqual(len(dataset), 1000)
class TrainerConfigurationTests(unittest.TestCase):
def test_the_basic_trainer_runs_ten_epochs_on_whatever_is_available(self) -> None:
trainer = quick_trainer_setup.basic_trainer()
self.assertEqual(trainer.max_epochs, 10)
self.assertGreaterEqual(trainer.num_devices, 1)
def test_the_debug_trainer_pins_itself_to_the_cpu(self) -> None:
# Debugging on an accelerator hides the errors you are debugging, and
# fast_dev_run is what makes it a debug configuration at all.
from lightning.pytorch.accelerators import CPUAccelerator
trainer = quick_trainer_setup.debug_trainer()
self.assertTrue(trainer.fast_dev_run)
self.assertIsInstance(trainer.accelerator, CPUAccelerator)
self.assertEqual(trainer.log_every_n_steps, 1)
def test_the_tuning_trainer_writes_nothing_and_halves_the_data(self) -> None:
trainer = quick_trainer_setup.hyperparameter_tuning_trainer(max_epochs=5)
self.assertEqual(trainer.max_epochs, 5)
self.assertEqual(trainer.checkpoint_callbacks, [])
self.assertIsNone(trainer.logger)
self.assertEqual(trainer.limit_train_batches, 0.5)
self.assertEqual(trainer.limit_val_batches, 0.5)
def test_the_overfit_trainer_restricts_itself_to_the_requested_batches(self) -> None:
trainer = quick_trainer_setup.overfit_test_trainer(num_batches=3)
self.assertEqual(trainer.overfit_batches, 3)
self.assertEqual(trainer.max_epochs, 100)
def test_the_production_trainer_checkpoints_stops_early_and_tracks_the_rate(self) -> None:
trainer = build_or_skip(self, quick_trainer_setup.production_single_gpu_trainer)
self.assertEqual(trainer.precision, "16-mixed")
self.assertEqual(trainer.gradient_clip_val, 1.0)
kinds = {type(callback) for callback in trainer.callbacks}
self.assertLessEqual(
{ModelCheckpoint, EarlyStopping, LearningRateMonitor}, kinds
)
checkpointer = trainer.checkpoint_callbacks[0]
self.assertEqual(checkpointer.monitor, "val/loss")
self.assertEqual(checkpointer.save_top_k, 3)
def test_the_time_limited_trainer_is_a_valid_checkpoint_configuration(self) -> None:
# The regression: save_top_k=3 with monitor=None made this function
# raise on every machine, GPU or not.
trainer = build_or_skip(
self, quick_trainer_setup.time_limited_trainer, max_time_hours=1.0
)
checkpointer = trainer.checkpoint_callbacks[0]
self.assertEqual(checkpointer.save_top_k, 3)
self.assertIsNotNone(checkpointer.monitor)
self.assertTrue(checkpointer.save_last)
def test_the_reproducible_trainer_seeds_and_pins_full_precision(self) -> None:
trainer = build_or_skip(self, quick_trainer_setup.reproducible_trainer, seed=123)
# seed_everything writes the seed where workers can read it back.
import os
self.assertEqual(os.environ.get("PL_GLOBAL_SEED"), "123")
self.assertEqual(trainer.precision, "32-true")
@unittest.skipUnless(CUDA_DEVICES >= 4, "DDP over 4 CUDA devices is unavailable")
def test_the_ddp_trainer_shards_across_the_requested_gpus(self) -> None:
from lightning.pytorch.strategies import DDPStrategy
trainer = quick_trainer_setup.multi_gpu_ddp_trainer(num_gpus=4)
self.assertIsInstance(trainer.strategy, DDPStrategy)
self.assertEqual(trainer.num_devices, 4)
@unittest.skipUnless(CUDA_DEVICES >= 8, "FSDP over 8 CUDA devices is unavailable")
def test_the_fsdp_trainer_uses_bfloat16_and_accumulates(self) -> None:
from lightning.pytorch.strategies import FSDPStrategy
trainer = quick_trainer_setup.large_model_fsdp_trainer(num_gpus=8)
self.assertIsInstance(trainer.strategy, FSDPStrategy)
self.assertEqual(trainer.precision, "bf16-mixed")
self.assertEqual(trainer.accumulate_grad_batches, 4)
def test_the_deepspeed_trainer_selects_the_requested_stage(self) -> None:
pytest.importorskip("deepspeed", reason="optional DeepSpeed is not installed")
if CUDA_DEVICES < 8:
self.skipTest("DeepSpeed stage 3 needs 8 CUDA devices")
trainer = quick_trainer_setup.deepspeed_trainer(stage=3)
self.assertEqual(trainer.accumulate_grad_batches, 4)
class TrainingLoopTests(unittest.TestCase):
"""One short CPU fit: the templates have to compose, not converge."""
@classmethod
def setUpClass(cls) -> None:
torch.manual_seed(0)
features = torch.randn(24, 784)
labels = torch.randint(0, 10, (24,))
cls.loader = DataLoader(TensorDataset(features, labels), batch_size=8)
cls.module = template_lightning_module.TemplateLightningModule(
learning_rate=0.01, hidden_dim=16, dropout=0.0
)
cls.before_weights = {name: value.detach().clone() for name, value in cls.module.named_parameters()}
cls._directory = tempfile.TemporaryDirectory()
cls.trainer = L.Trainer(
max_epochs=1,
accelerator="cpu",
devices=1,
logger=False,
enable_checkpointing=False,
enable_progress_bar=False,
enable_model_summary=False,
default_root_dir=cls._directory.name,
)
cls.trainer.fit(cls.module, cls.loader, cls.loader)
# Snapshot now: a later trainer.test() or .predict() replaces
# logged_metrics with that stage's own values.
cls.fit_metrics = dict(cls.trainer.logged_metrics)
@classmethod
def tearDownClass(cls) -> None:
cls._directory.cleanup()
def test_the_fit_logs_the_metrics_the_template_documents(self) -> None:
logged = set(self.fit_metrics)
self.assertIn("val/loss", logged)
self.assertIn("val/acc", logged)
self.assertIn("learning_rate", logged)
self.assertLessEqual({"train/loss_step", "train/loss_epoch"}, logged)
def test_the_logged_learning_rate_is_the_configured_one(self) -> None:
self.assertAlmostEqual(
float(self.fit_metrics["learning_rate"]), 0.01, places=6
)
def test_the_losses_are_finite_and_the_accuracies_are_proportions(self) -> None:
for name, value in self.fit_metrics.items():
with self.subTest(metric=name):
self.assertTrue(torch.isfinite(torch.as_tensor(value)))
if name.endswith("acc"):
self.assertGreaterEqual(float(value), 0.0)
self.assertLessEqual(float(value), 1.0)
def test_a_test_pass_reports_its_own_metrics(self) -> None:
results = self.trainer.test(self.module, self.loader, verbose=False)
self.assertEqual(set(results[0]), {"test/loss", "test/acc"})
def test_prediction_returns_one_class_index_per_sample(self) -> None:
predictions = self.trainer.predict(self.module, self.loader)
self.assertEqual(sum(len(batch) for batch in predictions), 24)
for batch in predictions:
self.assertEqual(batch.dtype, torch.int64)
self.assertTrue(bool(((batch >= 0) & (batch < 10)).all()))
def test_training_moved_the_weights(self) -> None:
# A fit that leaves every parameter untouched has not trained.
trained = dict(self.module.named_parameters())
self.assertTrue(any(not torch.allclose(value, trained[name])
for name, value in self.before_weights.items()))
if __name__ == "__main__":
unittest.main()