Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.
Python agent-skillsai-scientistbioinformaticschemoinformaticsclaudeclaude-skillsclaudecodeclinical-researchcomputational-biologydata-analysisdrug-discoverygenomicsmaterials-sciencemetabolomicsproteomicsscientific-computingscientific-visualization
"""Tests for the SHAP tabular report generator.
The report's one substantive claim is additivity: with an interventional
TreeExplainer in probability space, a row's base value plus its SHAP values must
reconstruct the model's predicted probability for that row. That identity is a
theorem about Shapley values, not a property of this code, which makes it the
right thing to assert -- and the script itself validates it, so the tests check
that the validation is real by verifying the reconstruction independently.
The rest guards the parts that silently produce a wrong report rather than
crashing: `select_output` picking the wrong class slice out of a
(samples, features, outputs) array, the CLI's numeric bounds accepting a zero
background, `save_axis` leaking figures, and the exported CSVs disagreeing with
the values they were built from.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
import pytest
import skill_contract
SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "shap"
SCRIPTS = SKILL_ROOT / "scripts"
sys.path.insert(0, str(SCRIPTS))
np = pytest.importorskip("numpy", reason="shap scripts need numpy")
pd = pytest.importorskip("pandas", reason="shap scripts need pandas")
shap = pytest.importorskip("shap", reason="shap scripts need shap")
pytest.importorskip("sklearn", reason="shap scripts need scikit-learn")
matplotlib = pytest.importorskip("matplotlib", reason="the report exports figures")
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from sklearn.datasets import load_breast_cancer # noqa: E402
from sklearn.ensemble import RandomForestClassifier # noqa: E402
import tabular_report # noqa: E402
CliHelpTests = skill_contract.cli.help_test_case(SKILL_ROOT)
REPORT = SCRIPTS / "tabular_report.py"
class ArgumentBoundTests(unittest.TestCase):
def test_a_positive_integer_is_accepted_and_zero_is_not(self) -> None:
# --background-size 0 would build an empty background, which makes every
# SHAP value meaningless rather than merely wrong.
self.assertEqual(tabular_report.positive_int("3"), 3)
for value in ("0", "-2"):
with self.subTest(value=value):
with self.assertRaises(argparse.ArgumentTypeError):
tabular_report.positive_int(value)
def test_zero_is_a_valid_class_index_but_a_negative_one_is_not(self) -> None:
self.assertEqual(tabular_report.nonnegative_int("0"), 0)
with self.assertRaises(argparse.ArgumentTypeError):
tabular_report.nonnegative_int("-1")
def test_tolerances_must_be_strictly_positive(self) -> None:
self.assertEqual(tabular_report.positive_float("1e-6"), 1e-6)
for value in ("0", "-1e-6", "nan", "inf", "-inf"):
with self.subTest(value=value):
with self.assertRaises(argparse.ArgumentTypeError):
tabular_report.positive_float(value)
class SingleOutputStub:
"""Stands in for an Explanation whose values have the given shape."""
def __init__(self, shape: tuple[int, ...]) -> None:
self.values = np.zeros(shape)
class OutputSelectionTests(unittest.TestCase):
def test_a_two_dimensional_explanation_is_returned_unchanged_for_class_zero(self) -> None:
stub = SingleOutputStub((4, 3))
self.assertIs(tabular_report.select_output(stub, 0), stub)
def test_a_two_dimensional_explanation_rejects_any_other_class(self) -> None:
with self.assertRaisesRegex(ValueError, "produced one output"):
tabular_report.select_output(SingleOutputStub((4, 3)), 1)
def test_an_unexpected_rank_names_the_shape_it_got(self) -> None:
with self.assertRaisesRegex(ValueError, r"received \(4,\)"):
tabular_report.select_output(SingleOutputStub((4,)), 0)
with self.assertRaisesRegex(ValueError, r"received \(2, 3, 4, 5\)"):
tabular_report.select_output(SingleOutputStub((2, 3, 4, 5)), 0)
def test_a_class_index_past_the_last_output_is_rejected(self) -> None:
with self.assertRaisesRegex(ValueError, "outside 2 outputs"):
tabular_report.select_output(SingleOutputStub((4, 3, 2)), 2)
def test_negative_output_index_cannot_silently_select_last_class(self) -> None:
with self.assertRaisesRegex(ValueError, "zero or greater"):
tabular_report.select_output(SingleOutputStub((4, 3, 2)), -1)
def test_output_names_fall_back_to_a_positional_label(self) -> None:
explanation = shap.Explanation(
values=np.zeros((2, 3)), base_values=np.zeros(2), output_names=None
)
self.assertEqual(tabular_report.output_name(explanation, 1), "class_1")
def test_a_named_output_keeps_its_name(self) -> None:
explanation = shap.Explanation(
values=np.zeros((2, 3)), base_values=np.zeros(2), output_names="malignant"
)
self.assertEqual(tabular_report.output_name(explanation, 0), "malignant")
class ExplanationTests(unittest.TestCase):
"""One small interventional TreeExplainer, reused across the assertions."""
@classmethod
def setUpClass(cls) -> None:
frame, target = load_breast_cancer(as_frame=True, return_X_y=True)
cls.features = frame.iloc[:120, :6]
cls.model = RandomForestClassifier(
n_estimators=20, min_samples_leaf=3, random_state=0
).fit(cls.features, target.iloc[:120])
background = shap.sample(cls.features, 20, random_state=0)
explainer = shap.TreeExplainer(
cls.model,
data=background,
feature_perturbation="interventional",
model_output="probability",
)
cls.explained = cls.features.iloc[:10]
cls.all_outputs = explainer(cls.explained)
def test_the_explainer_produces_one_slice_per_class(self) -> None:
self.assertEqual(
np.shape(self.all_outputs.values), (10, 6, len(self.model.classes_))
)
def test_selecting_a_class_takes_that_slice_and_no_other(self) -> None:
for class_index in range(len(self.model.classes_)):
with self.subTest(class_index=class_index):
selected = tabular_report.select_output(self.all_outputs, class_index)
np.testing.assert_allclose(
np.asarray(selected.values),
np.asarray(self.all_outputs.values)[:, :, class_index],
)
self.assertEqual(np.shape(selected.base_values), (10,))
def test_shap_values_reconstruct_the_predicted_probability(self) -> None:
# The defining property: base value + contributions = model output. The
# script asserts this with np.testing.assert_allclose; here the target
# is recomputed from predict_proba so the check is independent.
for class_index in range(len(self.model.classes_)):
with self.subTest(class_index=class_index):
selected = tabular_report.select_output(self.all_outputs, class_index)
reconstructed = np.asarray(selected.base_values, dtype=float) + np.asarray(
selected.values, dtype=float
).sum(axis=1)
predicted = self.model.predict_proba(self.explained)[:, class_index]
np.testing.assert_allclose(reconstructed, predicted, rtol=1e-5, atol=1e-6)
def test_the_two_class_slices_are_mirror_images(self) -> None:
# Binary probabilities sum to one, so each feature's contribution to
# class 1 must be the negative of its contribution to class 0.
values = np.asarray(self.all_outputs.values, dtype=float)
np.testing.assert_allclose(values[:, :, 0], -values[:, :, 1], atol=1e-9)
def test_the_base_value_is_the_background_mean_prediction(self) -> None:
# With an interventional masker the base value is the model's average
# output over the background sample, and it is identical for every row.
selected = tabular_report.select_output(self.all_outputs, 1)
base_values = np.asarray(selected.base_values, dtype=float)
np.testing.assert_allclose(base_values, base_values[0])
background = shap.sample(self.features, 20, random_state=0)
self.assertAlmostEqual(
float(base_values[0]),
float(self.model.predict_proba(background)[:, 1].mean()),
places=5,
)
class FigureExportTests(unittest.TestCase):
def setUp(self) -> None:
self.addCleanup(plt.close, "all")
def test_saving_an_axis_writes_a_file_and_leaves_no_open_figure(self) -> None:
# A report that exports four figures without closing them exhausts
# matplotlib's figure warning threshold and leaks memory.
figure, axis = plt.subplots()
axis.plot([0, 1], [0, 1])
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "figure.png"
tabular_report.save_axis(axis, path)
self.assertGreater(path.stat().st_size, 0)
self.assertEqual(plt.get_fignums(), [])
def test_a_missing_axis_falls_back_to_the_current_figure(self) -> None:
# Some shap.plots helpers return None and draw into gcf().
plt.figure()
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "current.png"
tabular_report.save_axis(None, path)
self.assertGreater(path.stat().st_size, 0)
self.assertEqual(plt.get_fignums(), [])
class ReportRunTests(unittest.TestCase):
"""One end-to-end CLI run, shared by the assertions about its output."""
@classmethod
def setUpClass(cls) -> None:
cls._directory = tempfile.TemporaryDirectory()
cls.output = Path(cls._directory.name) / "report"
completed = subprocess.run(
[
sys.executable, str(REPORT),
"--output-dir", str(cls.output),
"--background-size", "150",
"--explain-size", "12",
"--max-display", "5",
"--seed", "7",
],
capture_output=True,
text=True,
timeout=600,
)
if completed.returncode != 0:
raise AssertionError(
f"tabular_report.py exited {completed.returncode}:\n{completed.stderr}"
)
cls.stdout = completed.stdout
cls.metadata = json.loads((cls.output / "metadata.json").read_text())
@classmethod
def tearDownClass(cls) -> None:
cls._directory.cleanup()
def test_every_documented_artefact_is_written(self) -> None:
expected = {
"bar.png",
"beeswarm.png",
"feature_importance.csv",
"first_row_contributions.csv",
"metadata.json",
"prediction_reconstruction.csv",
"scatter-top-feature.png",
"waterfall-first-row.png",
}
self.assertEqual({path.name for path in self.output.iterdir()}, expected)
for path in self.output.iterdir():
with self.subTest(artefact=path.name):
self.assertGreater(path.stat().st_size, 0)
def test_the_metadata_records_the_run_it_actually_did(self) -> None:
self.assertEqual(self.metadata["seed"], 7)
self.assertEqual(self.metadata["background_rows"], 150)
self.assertEqual(self.metadata["background_requested_rows"], 150)
self.assertEqual(self.metadata["explained_rows"], 12)
# Breast cancer ships 30 features and 2 classes.
self.assertEqual(self.metadata["feature_count"], 30)
self.assertEqual(self.metadata["all_output_shape"], [12, 30, 2])
self.assertEqual(self.metadata["selected_output_shape"], [12, 30])
self.assertEqual(self.metadata["output_units"], "class probability")
self.assertEqual(self.metadata["feature_perturbation"], "interventional")
self.assertEqual(self.metadata["selected_class_meaning"], "benign")
self.assertEqual(self.metadata["selected_output_name"], "benign")
def test_the_additivity_error_is_inside_the_tolerance_it_reports(self) -> None:
self.assertLess(
self.metadata["max_abs_additivity_error"],
self.metadata["additivity_atol"] + self.metadata["additivity_rtol"],
)
def test_the_reconstruction_csv_adds_up_row_by_row(self) -> None:
frame = pd.read_csv(self.output / "prediction_reconstruction.csv")
self.assertEqual(len(frame), 12)
np.testing.assert_allclose(
frame["reconstructed_output"], frame["prediction"], atol=1e-6
)
np.testing.assert_allclose(
frame["reconstruction_error"],
frame["reconstructed_output"] - frame["prediction"],
atol=1e-12,
)
# Probability-space output: every prediction is a probability.
self.assertTrue(((frame["prediction"] >= 0) & (frame["prediction"] <= 1)).all())
np.testing.assert_allclose(
frame["base_value"], self.metadata["background_mean_prediction"], atol=1e-6
)
def test_the_importance_table_is_ranked_by_mean_absolute_shap(self) -> None:
frame = pd.read_csv(self.output / "feature_importance.csv")
self.assertEqual(len(frame), 30)
self.assertTrue(frame["mean_abs_shap"].is_monotonic_decreasing)
self.assertTrue((frame["mean_abs_shap"] >= 0).all())
# The signed mean can never exceed the absolute mean in magnitude.
self.assertTrue((frame["mean_signed_shap"].abs() <= frame["mean_abs_shap"] + 1e-12).all())
self.assertEqual(frame.loc[0, "feature"], self.metadata["top_feature"])
def test_the_first_row_breakdown_is_ordered_by_contribution_size(self) -> None:
frame = pd.read_csv(self.output / "first_row_contributions.csv")
self.assertEqual(len(frame), 30)
self.assertTrue(frame["shap_value"].abs().is_monotonic_decreasing)
# Those contributions are the ones that reconstruct row 0's prediction.
reconstruction = pd.read_csv(self.output / "prediction_reconstruction.csv")
self.assertAlmostEqual(
float(reconstruction.loc[0, "base_value"] + frame["shap_value"].sum()),
float(reconstruction.loc[0, "prediction"]),
places=5,
)
def test_the_run_prints_its_metadata_and_where_it_wrote(self) -> None:
self.assertIn("shap_version", self.stdout)
self.assertIn(str(self.output.resolve()), self.stdout)
class ReportFailureTests(unittest.TestCase):
def test_a_class_index_outside_the_model_is_rejected_before_any_work(self) -> None:
with tempfile.TemporaryDirectory() as directory:
completed = subprocess.run(
[
sys.executable, str(REPORT),
"--output-dir", directory,
"--class-index", "7",
"--background-size", "5",
"--explain-size", "5",
],
capture_output=True,
text=True,
timeout=600,
)
self.assertNotEqual(completed.returncode, 0)
self.assertIn("--class-index 7 is outside model classes", completed.stderr)
def test_an_upstream_additivity_failure_prevents_report_artifacts(self) -> None:
# This frozen 0.52.0 breast-cancer configuration encounters a native
# split-threshold discrepancy. The report must reject it, not export
# an apparently successful decomposition by loosening tolerances.
with tempfile.TemporaryDirectory() as directory:
completed = subprocess.run(
[sys.executable, str(REPORT), "--output-dir", directory,
"--seed", "3", "--background-size", "150", "--explain-size", "2"],
capture_output=True, text=True, timeout=120,
)
self.assertNotEqual(completed.returncode, 0)
self.assertIn("Not equal to tolerance", completed.stderr)
self.assertEqual(list(Path(directory).iterdir()), [])
def test_a_negative_background_size_is_refused_by_the_parser(self) -> None:
completed = subprocess.run(
[sys.executable, str(REPORT), "--background-size", "-1"],
capture_output=True,
text=True,
timeout=120,
)
self.assertEqual(completed.returncode, 2)
self.assertIn("greater than zero", completed.stderr)
if __name__ == "__main__":
unittest.main()