Skip to content

Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.

46.6 KB
Raw Blame
"""Unit tests for the analytical-method-validation skill scripts.

Everything runs offline; the scripts make no network calls. The statistical
implementations are checked against published quantiles and hand-checkable
cases, because the whole value of the skill rests on them being right.

    uv run --with pytest python -m pytest tests/analytical-method-validation -q
"""

from __future__ import annotations

import importlib.util
import json
import math
import subprocess
import sys
import unittest
from pathlib import Path

import skill_contract

SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "analytical-method-validation"
SCRIPTS_DIR = SKILL_ROOT / "scripts"
FIXTURES = Path(__file__).resolve().parent / "fixtures"


def _load_script(name: str):
    """Load a bundled script as a module regardless of cwd."""
    if str(SCRIPTS_DIR) not in sys.path:
        sys.path.insert(0, str(SCRIPTS_DIR))
    spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / f"{name}.py")
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module
    spec.loader.exec_module(module)
    return module


common = _load_script("_common")
catalog = _load_script("_catalog")


def run_script(name: str, *args: str) -> subprocess.CompletedProcess:
    """Invoke a script as a subprocess and capture its streams."""
    return subprocess.run(
        [sys.executable, str(SCRIPTS_DIR / f"{name}.py"), *args],
        capture_output=True,
        text=True,
        check=False,
    )


# ---------------------------------------------------------------- structure


class TestSkillStructure(unittest.TestCase):
    def test_skill_md_frontmatter(self):
        text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
        self.assertTrue(text.startswith("---\n"))
        end = text.index("\n---\n", 4)
        front = text[4:end]
        self.assertIn("name: analytical-method-validation", front)
        self.assertRegex(front, r'\n  version: "\d+\.\d+"\n')
        # allowed-tools must be a space-separated string, not a YAML list
        for line in front.splitlines():
            if line.startswith("allowed-tools:"):
                self.assertNotIn(",", line)
                self.assertNotIn("[", line)

    def test_skill_md_under_500_lines(self):
        lines = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8").splitlines()
        self.assertLess(len(lines), 500, f"SKILL.md has {len(lines)} lines")

    def test_referenced_files_exist(self):
        text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8")
        for rel in ("references/framework-selection.md", "references/ich-q2r2.md",
                    "references/ich-m10-bioanalytical.md", "references/compendial-and-clsi.md",
                    "references/statistics.md", "references/source-ledger.md",
                    "assets/validation-protocol-template.md",
                    "assets/validation-report-template.md"):
            self.assertIn(rel, text, f"{rel} not referenced from SKILL.md")
            self.assertTrue((SKILL_ROOT / rel).is_file(), f"{rel} missing")

    def test_no_bytecode_shipped(self):
        self.assertEqual(list(SKILL_ROOT.rglob("*.pyc")), [])
        self.assertEqual(list(SKILL_ROOT.rglob("__pycache__")), [])

    def test_no_tests_inside_skill(self):
        self.assertEqual(list(SKILL_ROOT.rglob("test_*.py")), [])

    def test_no_techniques_attributed_to_q2r2_that_it_never_mentions(self):
        """Guard against secondary-source claims leaking in as primary-source facts.

        Trade summaries of the Q2(R1) -> Q2(R2) revision commonly list Raman
        spectroscopy. The adopted guideline text does not contain the word, so the
        skill must not attribute it to Q2(R2).
        """
        for path in SKILL_ROOT.rglob("*.md"):
            text = path.read_text(encoding="utf-8")
            self.assertNotIn("Raman", text, f"{path.name} attributes Raman to ICH guidance")

    def test_paywalled_standards_are_not_quoted_with_numbers(self):
        """USP/CLSI/ISO content must be referenced, never reproduced.

        A numeric threshold sitting next to a paywalled designation is the shape of
        an accidental reproduction, so the reference file states scope only.
        """
        text = (SKILL_ROOT / "references" / "compendial-and-clsi.md").read_text(encoding="utf-8")
        self.assertIn("does not reproduce", text)
        for banned in ("tailing factor limit", "%RSD limit of", "must not exceed 2.0"):
            self.assertNotIn(banned, text)


# ---------------------------------------------------------------- distributions


class TestDistributions(unittest.TestCase):
    def test_t_quantiles(self):
        for p, df, want in [
            (0.975, 1, 12.706205),
            (0.975, 5, 2.570582),
            (0.975, 9, 2.262157),
            (0.95, 10, 1.812461),
            (0.995, 20, 2.845340),
            (0.975, 100, 1.983972),
        ]:
            self.assertAlmostEqual(common.t_ppf(p, df), want, places=4)

    def test_t_cdf_symmetry(self):
        self.assertAlmostEqual(common.t_cdf(0.0, 7), 0.5, places=10)
        self.assertAlmostEqual(
            common.t_cdf(-1.3, 12) + common.t_cdf(1.3, 12), 1.0, places=10
        )

    def test_chi2_quantiles(self):
        for p, df, want in [
            (0.025, 9, 2.700389),
            (0.975, 9, 19.022768),
            (0.95, 1, 3.841459),
            (0.05, 20, 10.850811),
            (0.5, 4, 3.356694),
        ]:
            self.assertAlmostEqual(common.chi2_ppf(p, df), want, places=3)

    def test_f_distribution(self):
        # F(1,1) upper 5% point is 161.4476
        self.assertAlmostEqual(common.f_sf(161.4476, 1, 1), 0.05, places=5)
        # median-ish symmetry check
        self.assertAlmostEqual(common.f_sf(1.0, 5, 5), 0.5, places=6)
        self.assertAlmostEqual(common.f_sf(4.0, 2, 10), 0.052922, places=5)
        self.assertAlmostEqual(common.f_sf(0.0, 3, 5), 1.0, places=12)

    def test_f_sf_does_not_underflow_to_zero(self):
        """A validation report must not print a lack-of-fit p-value of exactly 0.

        Computing 1 - cdf underflows well before the true tail probability does.
        """
        for f, df1, df2 in [(1e8, 3, 5), (1e3, 1, 200), (1e6, 10, 100)]:
            tail = common.f_sf(f, df1, df2)
            self.assertGreater(tail, 0.0, f"F={f} df=({df1},{df2}) underflowed")
            self.assertLess(tail, 1e-15)
        # And it still agrees with 1 - cdf where that is accurate.
        for f, df1, df2 in [(4.0, 2, 10), (2.5, 3, 12), (1.0, 5, 5)]:
            self.assertAlmostEqual(
                common.f_sf(f, df1, df2), 1.0 - common.f_cdf(f, df1, df2), places=12
            )

    def test_betainc_edges(self):
        self.assertEqual(common.betainc(2.0, 3.0, 0.0), 0.0)
        self.assertEqual(common.betainc(2.0, 3.0, 1.0), 1.0)
        # I_x(a,b) + I_(1-x)(b,a) == 1
        self.assertAlmostEqual(
            common.betainc(2.5, 4.5, 0.3) + common.betainc(4.5, 2.5, 0.7), 1.0, places=12
        )

    def test_bad_probabilities_raise(self):
        with self.assertRaises(common.InputError):
            common.t_ppf(0.0, 5)
        with self.assertRaises(common.InputError):
            common.chi2_ppf(1.0, 5)


# ---------------------------------------------------------------- regression


class TestRegression(unittest.TestCase):
    def test_exact_line(self):
        fit = common.fit_linear([1, 2, 3, 4], [3, 5, 7, 9])
        self.assertAlmostEqual(fit.slope, 2.0, places=12)
        self.assertAlmostEqual(fit.intercept, 1.0, places=12)
        self.assertAlmostEqual(fit.r_squared, 1.0, places=12)
        self.assertAlmostEqual(fit.residual_sd, 0.0, places=10)

    def test_known_least_squares(self):
        fit = common.fit_linear([1, 2, 3, 4, 5], [2.1, 4.0, 6.2, 7.9, 10.1])
        self.assertAlmostEqual(fit.slope, 1.99, places=10)
        self.assertAlmostEqual(fit.intercept, 0.09, places=10)
        lo, hi = fit.slope_ci()
        self.assertLess(lo, 1.99)
        self.assertGreater(hi, 1.99)

    def test_weighting_changes_the_fit(self):
        xs = [1, 2, 5, 10, 20]
        ys = [1.0, 2.1, 5.2, 9.5, 21.0]
        unweighted = common.fit_linear(xs, ys)
        weighted = common.fit_linear(xs, ys, [1.0 / (x * x) for x in xs])
        self.assertNotAlmostEqual(unweighted.slope, weighted.slope, places=6)

    def test_too_few_points(self):
        with self.assertRaises(common.InputError):
            common.fit_linear([1, 2], [1, 2])

    def test_collinear_x_raises(self):
        with self.assertRaises(common.InputError):
            common.fit_linear([3, 3, 3], [1, 2, 3])

    def test_lack_of_fit_detects_curvature(self):
        xs = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]
        quad = [x * x for x in xs]
        ys = [q + (0.05 if i % 2 else -0.05) for i, q in enumerate(quad)]
        lof = common.lack_of_fit(xs, ys, common.fit_linear(xs, ys))
        self.assertTrue(lof["applicable"])
        self.assertLess(lof["p_value"], 0.001)

    def test_lack_of_fit_passes_linear_data(self):
        xs = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5]
        ys = [2 * x + (0.02 if i % 2 else -0.02) for i, x in enumerate(xs)]
        lof = common.lack_of_fit(xs, ys, common.fit_linear(xs, ys))
        self.assertTrue(lof["applicable"])
        self.assertGreater(lof["p_value"], 0.05)

    def test_lack_of_fit_needs_replicates(self):
        xs = [1, 2, 3, 4, 5]
        ys = [2, 4, 6, 8, 10.1]
        lof = common.lack_of_fit(xs, ys, common.fit_linear(xs, ys))
        self.assertFalse(lof["applicable"])
        self.assertIn("replicates", lof["reason"])

    def test_runs_test_flags_curvature(self):
        xs = list(range(1, 13))
        ys = [x * x for x in xs]
        runs = common.runs_test(common.fit_linear(xs, ys).residuals)
        self.assertLess(runs["p_value"], 0.05)

    def test_runs_test_small_sample_is_honest(self):
        runs = common.runs_test([1.0, -1.0, 1.0])
        self.assertTrue(math.isnan(runs["p_value"]))
        self.assertIn("too few", runs["note"])

    def test_heteroscedasticity_ratio(self):
        xs = list(range(1, 13))
        resid = [0.01 * (1 if i % 2 else -1) for i in range(6)] + [
            1.0 * (1 if i % 2 else -1) for i in range(6)
        ]
        het = common.heteroscedasticity(xs, resid)
        self.assertTrue(het["applicable"])
        self.assertGreater(het["variance_ratio_high_over_low"], 100)


# ---------------------------------------------------------------- precision


class TestVarianceComponents(unittest.TestCase):
    def test_balanced_hand_check(self):
        groups = {"d1": [10.0, 10.2, 9.8], "d2": [10.5, 10.6, 10.4], "d3": [9.6, 9.5, 9.7]}
        comp = common.one_way_components(groups)
        self.assertEqual(comp.n_total, 9)
        self.assertEqual(comp.n_groups, 3)
        self.assertTrue(comp.balanced)
        self.assertAlmostEqual(comp.n_effective, 3.0, places=12)
        self.assertAlmostEqual(comp.ms_within, 0.02, places=12)
        self.assertAlmostEqual(comp.ms_between, 0.61, places=12)
        # s2_between = (0.61 - 0.02)/3
        self.assertAlmostEqual(comp.sd_between ** 2, (0.61 - 0.02) / 3.0, places=12)
        self.assertAlmostEqual(
            comp.sd_intermediate ** 2, 0.02 + (0.61 - 0.02) / 3.0, places=12
        )
        self.assertGreater(comp.sd_intermediate, comp.sd_repeatability)

    def test_between_variance_truncated_at_zero(self):
        # Groups deliberately indistinguishable: MS_between < MS_within
        groups = {"a": [1.0, 3.0, 2.0], "b": [2.0, 1.0, 3.0], "c": [3.0, 2.0, 1.0]}
        comp = common.one_way_components(groups)
        self.assertEqual(comp.sd_between, 0.0)
        self.assertAlmostEqual(comp.sd_intermediate, comp.sd_repeatability, places=12)

    def test_unbalanced_effective_n(self):
        groups = {"a": [1.0, 1.1, 0.9, 1.05], "b": [2.0, 2.1], "c": [3.0, 3.1, 2.9]}
        comp = common.one_way_components(groups)
        self.assertFalse(comp.balanced)
        self.assertGreater(comp.n_effective, 1.0)
        self.assertLess(comp.n_effective, 4.0)

    def test_needs_two_groups(self):
        with self.assertRaises(common.InputError):
            common.one_way_components({"only": [1.0, 2.0, 3.0]})

    def test_sd_confidence_interval_brackets(self):
        lo, hi = common.sd_confidence_interval(1.0, 9, 0.90)
        self.assertLess(lo, 1.0)
        self.assertGreater(hi, 1.0)
        # chi2 interval for sd=1, df=9 at 90%
        self.assertAlmostEqual(lo, math.sqrt(9 / common.chi2_ppf(0.95, 9)), places=10)


# ---------------------------------------------------------------- comparison


class TestMethodComparison(unittest.TestCase):
    def test_deming_recovers_unit_slope(self):
        xs = list(range(1, 21))
        ys = [x + (0.05 if i % 2 else -0.05) for i, x in enumerate(xs)]
        res = common.deming(xs, ys, 1.0)
        self.assertAlmostEqual(res["slope"], 1.0, places=2)
        self.assertAlmostEqual(res["intercept"], 0.0, places=1)

    def test_deming_lambda_affects_slope(self):
        xs = [1, 2, 3, 4, 5, 6, 7, 8]
        ys = [1.2, 1.9, 3.4, 3.8, 5.5, 5.9, 7.6, 7.9]
        a = common.deming(xs, ys, 1.0)["slope"]
        b = common.deming(xs, ys, 4.0)["slope"]
        self.assertNotAlmostEqual(a, b, places=6)

    def test_deming_lambda_direction_matches_documentation(self):
        """lambda = var(y error)/var(x error), pinned by the unambiguous limits.

        Large lambda means x is effectively error-free, so the fit must converge
        on the OLS slope of y on x. Small lambda means y is error-free, so it must
        converge on the inverse regression. Documenting this backwards would send
        a user with unequal precision in exactly the wrong direction.
        """
        xs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        ys = [1.2, 1.9, 3.4, 3.8, 5.5, 5.9, 7.6, 7.9, 9.3, 10.4]
        xb, yb = common.mean(xs), common.mean(ys)
        sxx = math.fsum((x - xb) ** 2 for x in xs)
        syy = math.fsum((y - yb) ** 2 for y in ys)
        sxy = math.fsum((x - xb) * (y - yb) for x, y in zip(xs, ys))
        ols_y_on_x = sxy / sxx
        inverse = syy / sxy
        self.assertAlmostEqual(common.deming(xs, ys, 1e9)["slope"], ols_y_on_x, places=6)
        self.assertAlmostEqual(common.deming(xs, ys, 1e-9)["slope"], inverse, places=6)
        # lambda = 1 is orthogonal regression, between the two.
        orth = common.deming(xs, ys, 1.0)["slope"]
        self.assertLess(ols_y_on_x, orth)
        self.assertLess(orth, inverse)

    def test_deming_rejects_bad_lambda(self):
        with self.assertRaises(common.InputError):
            common.deming([1, 2, 3], [1, 2, 3], 0.0)

    def test_passing_bablok_exact_line(self):
        xs = [1, 2, 3, 4, 5, 6, 7]
        ys = [2 * x + 1 for x in xs]
        res = common.passing_bablok(xs, ys)
        self.assertAlmostEqual(res["slope"], 2.0, places=10)
        self.assertAlmostEqual(res["intercept"], 1.0, places=10)

    def test_passing_bablok_resists_one_outlier(self):
        xs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        ys = [float(x) for x in xs]
        ys[4] = 50.0  # gross outlier
        pb = common.passing_bablok(xs, ys)
        ols = common.fit_linear(xs, ys)
        self.assertAlmostEqual(pb["slope"], 1.0, places=6)
        self.assertGreater(abs(ols.slope - 1.0), abs(pb["slope"] - 1.0))

    def test_passing_bablok_needs_five_points(self):
        with self.assertRaises(common.InputError):
            common.passing_bablok([1, 2, 3], [1, 2, 3])

    def test_passing_bablok_ci_order_statistics(self):
        """Pin the 1-based-to-0-based conversion of the M1/M2 order statistics.

        M1 and M2 are both 1-based positions in the shifted slope list, so both
        convert with the same -1. Applying the offset to only one end silently
        narrows the interval on the low side.
        """
        xs = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]
        deltas = [1.2, -0.8, 0.5, -1.5, 2.0, -0.3, 1.1, -2.2, 0.9, -1.0, 1.8, -0.6]
        ys = [1.05 * x + 2 + d for x, d in zip(xs, deltas)]
        res = common.passing_bablok(xs, ys)

        slopes = sorted(
            (ys[j] - ys[i]) / (xs[j] - xs[i])
            for i in range(len(xs))
            for j in range(i + 1, len(xs))
            if xs[j] != xs[i]
        )
        n_slopes = len(slopes)
        shift = sum(1 for s in slopes if s < -1.0)
        c = common.z_ppf(0.975) * math.sqrt(
            len(xs) * (len(xs) - 1.0) * (2.0 * len(xs) + 5.0) / 18.0
        )
        m1 = int(round((n_slopes - c) / 2.0))
        m2 = n_slopes - m1 + 1
        self.assertAlmostEqual(res["slope_ci95"][0], slopes[m1 + shift - 1], places=12)
        self.assertAlmostEqual(res["slope_ci95"][1], slopes[m2 + shift - 1], places=12)
        self.assertLessEqual(res["slope_ci95"][0], res["slope"])
        self.assertLessEqual(res["slope"], res["slope_ci95"][1])

    def test_passing_bablok_ci_contains_estimate_across_shapes(self):
        for n in (7, 8, 11, 12, 19, 20):
            xs = [10.0 + 5 * i for i in range(n)]
            ys = [1.03 * x + 1 + (0.7 if i % 3 == 0 else -0.4) for i, x in enumerate(xs)]
            res = common.passing_bablok(xs, ys)
            lo, hi = res["slope_ci95"]
            self.assertLessEqual(lo, res["slope"], f"n={n}")
            self.assertLessEqual(res["slope"], hi, f"n={n}")

    def test_bland_altman_bias_and_loa(self):
        xs = [10.0] * 6
        ys = [11.0, 11.0, 11.0, 11.0, 11.0, 11.0]
        ba = common.bland_altman(xs, ys)
        self.assertAlmostEqual(ba["bias"], 1.0, places=12)
        self.assertAlmostEqual(ba["sd_differences"], 0.0, places=12)

    def test_bland_altman_relative(self):
        ba = common.bland_altman([100.0, 200.0, 400.0], [110.0, 220.0, 440.0], relative=True)
        # Each pair is a 10% increase: (110-100)/105 = 9.5238% of the pair mean.
        self.assertAlmostEqual(ba["bias"], 9.523809523, places=6)
        self.assertAlmostEqual(ba["sd_differences"], 0.0, places=9)

    def test_bland_altman_needs_three_pairs(self):
        with self.assertRaises(common.InputError):
            common.bland_altman([100.0, 200.0], [110.0, 220.0])

    def test_tost_equivalent(self):
        res = common.tost_paired([0.1, -0.2, 0.05, 0.0, 0.15], margin=2.0)
        self.assertTrue(res["equivalent"])
        self.assertLess(res["p_value"], 0.05)

    def test_tost_not_equivalent_when_biased(self):
        res = common.tost_paired([1.9, 2.1, 2.0, 1.95, 2.05], margin=2.0)
        self.assertFalse(res["equivalent"])
        self.assertGreater(res["p_value"], 0.05)

    def test_tost_ci_is_1_minus_2alpha(self):
        diffs = [0.5, 0.6, 0.4, 0.55, 0.45, 0.5]
        res = common.tost_paired(diffs, margin=1.0, alpha=0.05)
        lo, hi = res["ci_1_minus_2alpha"]
        sd = common.sample_sd(diffs)
        se = sd / math.sqrt(len(diffs))
        expected_half = common.t_ppf(0.95, len(diffs) - 1) * se
        self.assertAlmostEqual(hi - res["mean_difference"], expected_half, places=10)
        self.assertAlmostEqual(res["mean_difference"] - lo, expected_half, places=10)

    def test_tost_rejects_bad_margin(self):
        with self.assertRaises(common.InputError):
            common.tost_paired([0.1, 0.2], margin=0.0)


# ---------------------------------------------------------------- catalogue


class TestCatalogue(unittest.TestCase):
    def test_m10_modalities_differ(self):
        cc = catalog.M10_CRITERIA["chromatographic"]
        lba = catalog.M10_CRITERIA["lba"]
        self.assertEqual(cc["accuracy_tolerance_pct"], 15.0)
        self.assertEqual(lba["accuracy_tolerance_pct"], 20.0)
        self.assertEqual(cc["accuracy_tolerance_lloq_pct"], 20.0)
        self.assertEqual(lba["accuracy_tolerance_lloq_pct"], 25.0)
        self.assertEqual(cc["isr_tolerance_pct"], 20.0)
        self.assertEqual(lba["isr_tolerance_pct"], 30.0)

    def test_total_error_is_lba_only(self):
        self.assertIsNone(catalog.M10_CRITERIA["chromatographic"]["total_error_pct"])
        self.assertEqual(catalog.M10_CRITERIA["lba"]["total_error_pct"], 30.0)
        self.assertEqual(catalog.M10_CRITERIA["lba"]["total_error_pct_at_limits"], 40.0)

    def test_m10_run_structure(self):
        cc = catalog.M10_CRITERIA["chromatographic"]
        lba = catalog.M10_CRITERIA["lba"]
        self.assertEqual(cc["qc_levels_accuracy_precision"], 4)
        self.assertEqual(lba["qc_levels_accuracy_precision"], 5)
        self.assertEqual(cc["ap_replicates_per_run"], 5)
        self.assertEqual(lba["ap_replicates_per_run"], 3)
        self.assertEqual(cc["ap_min_runs"], 3)
        self.assertEqual(lba["ap_min_runs"], 6)
        for crit in (cc, lba):
            self.assertEqual(crit["calibration_min_levels"], 6)
            self.assertAlmostEqual(crit["calibration_min_pass_fraction"], 0.75)
            self.assertAlmostEqual(crit["isr_pass_fraction"], 2 / 3)

    def test_q2r2_table1_limit_test_needs_dl_not_ql(self):
        limit = catalog.Q2R2_TESTS_BY_ATTRIBUTE["impurity-limit"]
        quant = catalog.Q2R2_TESTS_BY_ATTRIBUTE["impurity-quantitative"]
        self.assertEqual(limit["lower-range-limit"], "required-DL")
        self.assertEqual(quant["lower-range-limit"], "required-QL")
        self.assertEqual(limit["accuracy"], "not-normally")
        self.assertEqual(quant["accuracy"], "required")

    def test_q2r2_identity_only_needs_specificity(self):
        ident = catalog.Q2R2_TESTS_BY_ATTRIBUTE["identity"]
        self.assertEqual(ident["specificity"], "required")
        for key, value in ident.items():
            if key != "specificity":
                self.assertEqual(value, "not-normally")

    def test_attribute_aliases_resolve(self):
        self.assertEqual(catalog.resolve_attribute("Content"), "assay")
        self.assertEqual(catalog.resolve_attribute("related-substances"), "impurity-quantitative")
        self.assertEqual(catalog.resolve_attribute("limit-test"), "impurity-limit")
        with self.assertRaises(KeyError):
            catalog.resolve_attribute("nonsense")

    def test_paywalled_frameworks_marked_not_reproducible(self):
        for key in ("usp-1220", "usp-1225", "usp-1226", "clsi", "iso-17025"):
            self.assertFalse(catalog.FRAMEWORKS[key]["reproducible"], key)
        for key in ("ich-q2r2", "ich-m10"):
            self.assertTrue(catalog.FRAMEWORKS[key]["reproducible"], key)

    def test_q2r2_response_requires_five_levels(self):
        self.assertIn("5", catalog.Q2R2_STUDY_DESIGN["response"]["requirement"])

    def test_dl_ql_factors_documented(self):
        sd_slope = catalog.DL_QL_APPROACHES["sd-and-slope"]
        self.assertIn("3.3", sd_slope["dl"])
        self.assertIn("10", sd_slope["ql"])


# ---------------------------------------------------------------- CLI


class TestPlanValidation(unittest.TestCase):
    def test_list_frameworks(self):
        res = run_script("plan_validation.py".removesuffix(".py"), "--list-frameworks",
                         "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)
        rows = json.loads(res.stdout)
        keys = {r["key"] for r in rows}
        self.assertIn("ich-q2r2", keys)
        self.assertIn("ich-m10", keys)

    def test_q2r2_assay_plan(self):
        res = run_script("plan_validation", "--framework", "ich-q2r2",
                         "--attribute", "assay", "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)
        rows = json.loads(res.stdout)
        chars = {r["characteristic"] for r in rows}
        self.assertIn("specificity", chars)
        self.assertIn("intermediate-precision", chars)
        self.assertIn("robustness", chars)

    def test_identity_plan_marks_most_tests_not_required(self):
        res = run_script("plan_validation", "--framework", "ich-q2r2",
                         "--attribute", "identity", "--format", "json")
        rows = {r["characteristic"]: r["required"] for r in json.loads(res.stdout)}
        self.assertEqual(rows["specificity"], "conduct")
        self.assertIn("not normally", rows["accuracy"])

    def test_m10_requires_modality(self):
        res = run_script("plan_validation", "--framework", "ich-m10")
        self.assertEqual(res.returncode, 2)
        self.assertIn("modality", res.stderr)

    def test_m10_lba_plan_includes_total_error(self):
        res = run_script("plan_validation", "--framework", "ich-m10",
                         "--modality", "lba", "--format", "json")
        items = {r["item"] for r in json.loads(res.stdout)}
        self.assertIn("total error", items)

    def test_m10_chromatographic_plan_excludes_total_error(self):
        res = run_script("plan_validation", "--framework", "ich-m10",
                         "--modality", "chromatographic", "--format", "json")
        items = {r["item"] for r in json.loads(res.stdout)}
        self.assertNotIn("total error", items)

    def test_unknown_framework_exits_2(self):
        res = run_script("plan_validation", "--framework", "nope")
        self.assertEqual(res.returncode, 2)

    def test_protocol_skeleton_has_pre_stated_criteria_section(self):
        res = run_script("plan_validation", "--framework", "ich-q2r2",
                         "--attribute", "impurity", "--protocol")
        self.assertEqual(res.returncode, 0, res.stderr)
        self.assertIn("Pre-stated acceptance criteria", res.stdout)
        self.assertIn("BEFORE data collection", res.stdout)

    def test_paywalled_framework_refuses_to_supply_design(self):
        res = run_script("plan_validation", "--framework", "usp-1225", "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)
        self.assertIn("copyrighted", res.stderr)
        rows = json.loads(res.stdout)
        self.assertIn("obtain_from", rows[0])


class TestCheckResponse(unittest.TestCase):
    def test_good_curve_no_findings(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_good.csv"),
                         "--max-back-calc-error", "2")
        self.assertEqual(res.returncode, 0, res.stderr)

    def test_curved_response_flagged(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_curved.csv"),
                         "--max-back-calc-error", "2")
        self.assertEqual(res.returncode, 1)
        self.assertIn("lack-of-fit", res.stderr)

    def test_curved_response_has_high_r_squared(self):
        """The whole point: r-squared passes while the model is unusable."""
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_curved.csv"),
                         "--format", "json")
        payload = json.loads(res.stdout)[0]
        self.assertGreater(payload["summary"]["coefficient of determination (r2)"], 0.98)
        self.assertLess(payload["summary"]["lack-of-fit p"], 0.001)
        worst = max(abs(r["relative_error_pct"]) for r in payload["levels"])
        self.assertGreater(worst, 5.0)

    def test_too_few_levels_flagged(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_three_levels.csv"))
        self.assertEqual(res.returncode, 1)
        self.assertIn("recommends at least 5", res.stderr)

    def test_missing_column_exits_2(self):
        res = run_script("check_response", "-i", str(FIXTURES / "bad_columns.csv"))
        self.assertEqual(res.returncode, 2)
        self.assertIn("missing required column", res.stderr)

    def test_weighting_option_accepted(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_good.csv"),
                         "--weight", "1/x2", "--format", "json")
        payload = json.loads(res.stdout)[0]
        self.assertEqual(payload["summary"]["weighting"], "1/x2")


class TestCheckAccuracyPrecision(unittest.TestCase):
    def test_between_day_shift_inflates_intermediate_precision(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_between_day.csv"),
                         "--format", "json")
        payload = json.loads(res.stdout)[0]
        rows = payload["precision"]
        rep = [r for r in rows if r["level"] == "100"
               and r["component"].startswith("repeatability")][0]
        inter = [r for r in rows if r["level"] == "100"
                 and r["component"].startswith("intermediate")][0]
        self.assertGreater(inter["rsd_pct"], 10 * rep["rsd_pct"])

    def test_precision_is_reported_per_level(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_between_day.csv"),
                         "--format", "json")
        rows = json.loads(res.stdout)[0]["precision"]
        levels = {r["level"] for r in rows}
        self.assertIn("80", levels)
        self.assertIn("100", levels)
        self.assertIn("120", levels)
        # Per-level RSD must not be inflated by the 80/100/120 range itself.
        for r in rows:
            if r["component"].startswith("repeatability"):
                self.assertLess(r["rsd_pct"], 5.0)

    def test_accuracy_reports_confidence_interval(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_between_day.csv"),
                         "--format", "json")
        acc = json.loads(res.stdout)[0]["accuracy"]
        for row in acc:
            self.assertIn("ci95_low", row)
            self.assertIn("ci95_high", row)
            self.assertLess(row["ci95_low"], row["mean_recovery_pct"])
            self.assertGreater(row["ci95_high"], row["mean_recovery_pct"])

    def test_require_ci_within_limit_is_stricter(self):
        lenient = run_script("check_accuracy_precision", "-i",
                             str(FIXTURES / "ap_between_day.csv"), "--accuracy-limit", "1.0")
        strict = run_script("check_accuracy_precision", "-i",
                            str(FIXTURES / "ap_between_day.csv"), "--accuracy-limit", "1.0",
                            "--require-ci-within-limit")
        # The mean is inside +/-1%, so lenient passes; the CI is not, so strict fails.
        self.assertEqual(lenient.returncode, 0, lenient.stderr)
        self.assertEqual(strict.returncode, 1)
        self.assertIn("CI", strict.stderr)

    def test_no_group_column_only_repeatability(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_no_group.csv"),
                         "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)
        rows = json.loads(res.stdout)[0]["precision"]
        self.assertTrue(all("repeatability" in r["component"] for r in rows))
        self.assertIn("intermediate precision", res.stderr)

    def test_zero_level_rejected(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_zero_level.csv"))
        self.assertEqual(res.returncode, 2)

    def test_six_at_one_level_satisfies_option_b(self):
        """Q2(R2) 3.3.2.1 offers two alternatives; either is sufficient.

        Six determinations at 100% of the test concentration is option (b).
        Demanding nine unconditionally raises a finding against a design the
        guideline explicitly permits.
        """
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_no_group.csv"),
                         "--design-check", "assay", "--test-concentration", "100")
        self.assertEqual(res.returncode, 0, res.stderr)
        self.assertIn("option (b)", res.stderr)

    def test_nine_across_range_satisfies_option_a(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_between_day.csv"),
                         "--design-check", "assay")
        self.assertIn("option (a)", res.stderr)

    def test_design_meeting_neither_option_is_flagged(self):
        res = run_script("check_accuracy_precision", "-i", str(FIXTURES / "ap_thin_design.csv"),
                         "--design-check", "assay")
        self.assertEqual(res.returncode, 1)
        self.assertIn("neither", res.stderr)


class TestCheckDetectionLimits(unittest.TestCase):
    def test_multiple_approaches_reported(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--blanks", str(FIXTURES / "blanks.csv"), "--format", "json")
        payload = json.loads(res.stdout)[0]
        approaches = [e["approach"] for e in payload["estimates"]]
        self.assertEqual(len(approaches), 2)
        self.assertFalse(any("intercept" in a for a in approaches))
        self.assertTrue(any("blanks" in a for a in approaches))
        self.assertTrue(any("residual SD" in a for a in approaches))

    def test_dl_ql_factor_relationship(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"), "--format", "json")
        for est in json.loads(res.stdout)[0]["estimates"]:
            if math.isfinite(est["sigma"]):
                self.assertAlmostEqual(est["QL"] / est["DL"], 10.0 / 3.3, places=6)

    def test_ql_above_reporting_threshold_flagged(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--confirm-ql", "0.10", "--reporting-threshold", "0.05")
        self.assertEqual(res.returncode, 1)
        self.assertIn("reporting threshold", res.stderr)

    def test_ql_confirmation_evaluated(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--confirm-ql", "0.05",
                         "--confirm-data", str(FIXTURES / "ql_confirmation.csv"),
                         "--confirm-accuracy-limit", "10", "--confirm-rsd-limit", "10",
                         "--format", "json")
        payload = json.loads(res.stdout)[0]
        metrics = {r["metric"]: r["value"] for r in payload["confirmation"]}
        self.assertIn("bias vs claimed QL (%)", metrics)
        self.assertIn("RSD (%)", metrics)

    def test_confirm_data_without_claim_exits_2(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--confirm-data", str(FIXTURES / "ql_confirmation.csv"))
        self.assertEqual(res.returncode, 2)

    def test_threshold_check_uses_the_conservative_estimate(self):
        """With no claimed QL, the compliance check must not pick the flattering estimate.

        The fixture's QL estimates straddle 0.010. Taking the smallest would report
        no findings on a procedure whose conservative limit fails.
        """
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--blanks", str(FIXTURES / "blanks.csv"),
                         "--reporting-threshold", "0.010", "--format", "json")
        payload = json.loads(res.stdout)[0]
        qls = [e["QL"] for e in payload["estimates"]]
        self.assertLess(min(qls), 0.010)
        self.assertGreater(max(qls), 0.010)
        self.assertEqual(res.returncode, 1)
        self.assertIn("straddle", res.stderr)
        self.assertIn("conservative", res.stderr)

    def test_explicit_claim_overrides_the_conservative_default(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--blanks", str(FIXTURES / "blanks.csv"),
                         "--confirm-ql", "0.005", "--reporting-threshold", "0.010")
        self.assertNotIn("straddle", res.stderr)
        self.assertEqual(res.returncode, 1, res.stderr)
        self.assertIn("no confirmation data", res.stderr)

    def test_signal_to_noise_requires_level(self):
        res = run_script("check_detection_limits", "--calibration",
                         str(FIXTURES / "lowrange_calibration.csv"),
                         "--signal-to-noise", "12")
        self.assertEqual(res.returncode, 2)


class TestCheckBioanalyticalRun(unittest.TestCase):
    def test_per_level_qc_rule_catches_failing_level(self):
        res = run_script("check_bioanalytical_run", "--modality", "chromatographic",
                         "--run", str(FIXTURES / "m10_run_high_qc_fails.csv"))
        self.assertEqual(res.returncode, 1)
        self.assertIn("50% at each level", res.stderr)

    def test_same_run_passes_lba_tolerances(self):
        """The identical run passes under LBA tolerances -- the modality matters."""
        cc = run_script("check_bioanalytical_run", "--modality", "chromatographic",
                        "--run", str(FIXTURES / "m10_run_high_qc_fails.csv"))
        lba = run_script("check_bioanalytical_run", "--modality", "lba",
                         "--run", str(FIXTURES / "m10_run_high_qc_fails.csv"))
        self.assertEqual(cc.returncode, 1)
        self.assertEqual(lba.returncode, 0, lba.stderr)

    def test_modality_is_mandatory(self):
        res = run_script("check_bioanalytical_run", "--run",
                         str(FIXTURES / "m10_run_high_qc_fails.csv"))
        self.assertNotEqual(res.returncode, 0)
        self.assertIn("modality", res.stderr.lower())

    def test_isr_chromatographic_vs_lba(self):
        cc = run_script("check_bioanalytical_run", "--modality", "chromatographic",
                        "--isr", str(FIXTURES / "m10_isr.csv"))
        lba = run_script("check_bioanalytical_run", "--modality", "lba",
                         "--isr", str(FIXTURES / "m10_isr.csv"))
        # Differences sit between 20% and 30%, so this fails CC and passes LBA.
        self.assertEqual(cc.returncode, 1)
        self.assertEqual(lba.returncode, 0, lba.stderr)

    def test_total_error_rejected_for_chromatographic(self):
        res = run_script("check_bioanalytical_run", "--modality", "chromatographic",
                         "--total-error", str(FIXTURES / "m10_total_error.csv"))
        self.assertEqual(res.returncode, 2)
        self.assertIn("ligand binding", res.stderr)

    def test_total_error_applied_for_lba(self):
        res = run_script("check_bioanalytical_run", "--modality", "lba",
                         "--total-error", str(FIXTURES / "m10_total_error.csv"),
                         "--format", "json")
        rows = json.loads(res.stdout)[0]["total_error"]
        lloq = [r for r in rows if r["label"].lower() == "lloq"][0]
        mid = [r for r in rows if r["label"].lower() == "medium"][0]
        self.assertEqual(lloq["limit_pct"], 40.0)
        self.assertEqual(mid["limit_pct"], 30.0)

    def test_anchor_points_excluded_from_calibration_count(self):
        res = run_script("check_bioanalytical_run", "--modality", "lba",
                         "--run", str(FIXTURES / "m10_run_lba_anchor.csv"),
                         "--format", "json")
        rows = json.loads(res.stdout)[0]["run"]
        anchors = [r for r in rows if r["label"] == "ANCHOR"]
        self.assertTrue(anchors)
        self.assertEqual(anchors[0]["within"], "excluded")

    def test_criteria_listing(self):
        res = run_script("check_bioanalytical_run", "--modality", "lba", "--criteria",
                         "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)
        names = {r["criterion"] for r in json.loads(res.stdout)}
        self.assertIn("total_error_pct", names)

    def test_bad_type_column_exits_2(self):
        res = run_script("check_bioanalytical_run", "--modality", "chromatographic",
                         "--run", str(FIXTURES / "m10_run_bad_type.csv"))
        self.assertEqual(res.returncode, 2)


class TestCompareMethods(unittest.TestCase):
    def test_equivalent_at_two_percent(self):
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_paired.csv"),
                         "--margin", "2", "--relative", "--format", "json")
        payload = json.loads(res.stdout)[0]
        self.assertTrue(payload["tost"]["equivalent"])

    def test_not_equivalent_at_one_percent(self):
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_paired.csv"),
                         "--margin", "1", "--relative")
        self.assertEqual(res.returncode, 1)
        self.assertIn("equivalence NOT demonstrated", res.stderr)

    def test_t_test_significant_while_tost_equivalent(self):
        """The core teaching case: both statements are true and only one answers the question."""
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_paired.csv"),
                         "--margin", "2", "--relative", "--format", "json")
        payload = json.loads(res.stdout)[0]
        self.assertLess(payload["paired_t_p_value"], 0.05)
        self.assertTrue(payload["tost"]["equivalent"])

    def test_deming_and_ols_both_reported(self):
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_paired.csv"),
                         "--margin", "5", "--relative", "--format", "json")
        payload = json.loads(res.stdout)[0]
        self.assertIn("ols_slope", payload)
        self.assertIn("slope", payload["deming"])
        self.assertIn("slope", payload["passing_bablok"])

    def test_margin_is_required(self):
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_paired.csv"))
        self.assertEqual(res.returncode, 2)

    def test_too_few_pairs_exits_2(self):
        res = run_script("compare_methods", "-i", str(FIXTURES / "transfer_two_pairs.csv"),
                         "--margin", "2")
        self.assertEqual(res.returncode, 2)


# ---------------------------------------------------------------- I/O


class TestInputHandling(unittest.TestCase):
    def test_json_csv_and_tsv_normalize_the_same_record(self):
        record = {" level ": 0, " response ": " 10100 ", " sample ": " sample A ", " note ": None}
        expected = [{"level": "0", "response": "10100", "sample": "sample A", "note": ""}]
        for payload in ([record], {"rows": [record]}, {"data": [record]}):
            with self.subTest(payload=payload):
                rows = common.parse_rows(json.dumps(payload))
                self.assertEqual(rows, expected)
                common.require_columns(rows, ["level", "response"])
        for delimiter in (",", "\t"):
            with self.subTest(delimiter=delimiter):
                text = delimiter.join(record) + "\n"
                text += delimiter.join(("0", " 10100 ", " sample A ", "")) + "\n"
                self.assertEqual(common.parse_rows(text), expected)

    def test_padded_json_produces_the_same_analysis_as_csv(self):
        rows = common.parse_rows((FIXTURES / "calibration_good.csv").read_text())
        padded = [{f" {key} ": f" {value} " for key, value in row.items()} for row in rows]
        result = subprocess.run(
            [sys.executable, str(SCRIPTS_DIR / "check_response.py"), "-i", "-", "--format", "json"],
            input=json.dumps(padded), capture_output=True, text=True, check=False,
        )
        baseline = run_script("check_response", "-i", str(FIXTURES / "calibration_good.csv"),
                              "--format", "json")
        self.assertEqual(baseline.returncode, 0, baseline.stderr)
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertEqual(json.loads(result.stdout), json.loads(baseline.stdout))

    def test_json_input_accepted(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_good.json"),
                         "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)

    def test_tsv_input_accepted(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_good.tsv"),
                         "--format", "json")
        self.assertEqual(res.returncode, 0, res.stderr)

    def test_non_numeric_value_names_the_row(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_nonnumeric.csv"))
        self.assertEqual(res.returncode, 2)
        self.assertIn("not numeric", res.stderr)

    def test_ragged_csv_and_tsv_name_the_data_row(self):
        for delimiter, suffix in ((",", ".csv"), ("\t", ".tsv")):
            with self.subTest(suffix=suffix):
                text = delimiter.join(("level", "response")) + "\n"
                text += delimiter.join(("50", "10100")) + "\n"
                text += delimiter.join(("100", "20100", "extra")) + "\n"
                with self.assertRaisesRegex(
                    common.InputError, "row 2: more fields than the header"
                ):
                    common.parse_rows(text, path_hint="calibration" + suffix)

    def test_ragged_and_short_rows_are_cli_input_errors(self):
        for row, message in (("100,20100,extra", "more fields than the header"),
                             ("100", "not numeric")):
            with self.subTest(row=row):
                res = subprocess.run(
                    [sys.executable, str(SCRIPTS_DIR / "check_response.py"), "-i", "-"],
                    input="level,response\n50,10100\n" + row + "\n",
                    capture_output=True, text=True, check=False,
                )
                self.assertEqual(res.returncode, 2, res.stderr)
                self.assertIn(message, res.stderr)
                self.assertNotIn("Traceback", res.stderr)
                self.assertEqual(res.stdout, "")

    def test_missing_file_exits_2(self):
        res = run_script("check_response", "-i", str(FIXTURES / "does_not_exist.csv"))
        self.assertEqual(res.returncode, 2)

    def test_oversize_json_is_refused_not_truncated(self):
        """Silently dropping rows would give a clean result computed on part of the study."""
        payload = [{"level": 50 + (i % 5) * 25, "response": 10000 + i} for i in range(common.MAX_ROWS + 5)]
        with self.assertRaises(common.InputError) as ctx:
            common.parse_rows(json.dumps(payload))
        self.assertIn("more than", str(ctx.exception))

    def test_json_row_cap_boundary_is_accepted(self):
        payload = [{"level": 50 + (i % 5) * 25, "response": 10000 + i} for i in range(common.MAX_ROWS)]
        rows = common.parse_rows(json.dumps(payload))
        self.assertEqual(len(rows), common.MAX_ROWS)

    def test_empty_json_array_rejected(self):
        with self.assertRaises(common.InputError):
            common.parse_rows("[]")

    def test_omitted_input_does_not_block_on_a_terminal(self):
        """An omitted --input must error, not wait forever on a tty."""
        import unittest.mock as mock

        with mock.patch.object(sys, "stdin") as fake:
            fake.isatty.return_value = True
            with self.assertRaises(common.InputError) as ctx:
                common.read_input(None)
        self.assertIn("--input", str(ctx.exception))
        fake.read.assert_not_called()

    def test_stdout_carries_data_stderr_carries_notes(self):
        res = run_script("check_response", "-i", str(FIXTURES / "calibration_good.csv"),
                         "--format", "tsv")
        self.assertIn("statistic", res.stdout)
        self.assertNotIn("note:", res.stdout)
        self.assertIn("note:", res.stderr)


# The shared --help contract: every argparse CLI this skill ships answers --help
# without doing any work. It skips when the skill's packages are absent and runs
# for real under `python tests/run_all.py --isolated`.
CliHelpTests = skill_contract.cli.help_test_case(SKILL_ROOT)

if __name__ == "__main__":
    unittest.main()