Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.
Python agent-skillsai-scientistbioinformaticschemoinformaticsclaudeclaude-skillsclaudecodeclinical-researchcomputational-biologydata-analysisdrug-discoverygenomicsmaterials-sciencemetabolomicsproteomicsscientific-computingscientific-visualization
"""Dependency-free synthetic tests for Geniml skill helper CLIs."""
from __future__ import annotations
import ast
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
SKILL_ROOT = Path(__file__).resolve().parents[2] / "skills" / "geniml"
SCRIPTS = SKILL_ROOT / "scripts"
CLI_NAMES = (
"bed_validator.py",
"corpus_auditor.py",
"tokenizer_compatibility.py",
"consensus_plan.py",
"model_artifact_inspector.py",
"embedding_plan.py",
)
def run_cli(name: str, *arguments: str, cwd: Path | None = None):
command = [sys.executable, "-B", str(SCRIPTS / name), *map(str, arguments)]
return subprocess.run(
command,
cwd=cwd,
text=True,
capture_output=True,
timeout=20,
check=False,
)
def json_output(completed: subprocess.CompletedProcess[str]) -> dict:
try:
return json.loads(completed.stdout)
except json.JSONDecodeError as exc:
raise AssertionError(
f"invalid JSON\nstdout={completed.stdout}\nstderr={completed.stderr}"
) from exc
class ScriptHelpTests(unittest.TestCase):
def test_all_dependency_free_clis_have_help(self):
for name in CLI_NAMES:
with self.subTest(name=name):
completed = run_cli(name, "--help")
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertIn("usage:", completed.stdout.lower())
def test_scripts_parse_and_forbid_dangerous_capabilities(self):
forbidden_import_roots = {
"boto3",
"geniml",
"gtars",
"httpx",
"huggingface_hub",
"pickle",
"requests",
"socket",
"torch",
"urllib",
}
forbidden_calls = {"eval", "exec", "__import__"}
for path in sorted(SCRIPTS.glob("*.py")):
with self.subTest(path=path.name):
source = path.read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(path))
imported_roots = set()
called_names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imported_roots.update(
alias.name.split(".", 1)[0] for alias in node.names
)
elif isinstance(node, ast.ImportFrom) and node.module:
imported_roots.add(node.module.split(".", 1)[0])
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
called_names.add(node.func.id)
self.assertFalse(imported_roots & forbidden_import_roots)
self.assertFalse(called_names & forbidden_calls)
self.assertNotIn("os.getenv", source)
self.assertNotIn("os.environ", source)
class BedValidatorTests(unittest.TestCase):
def test_valid_bed_and_normalization_plan(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / "chrom.sizes").write_text("chr1\t1000\nchr2\t500\n")
(root / "valid.bed").write_text(
"chr1\t0\t10\tpeak1\t0\t+\n"
"chr1\t20\t30\tpeak2\t0\t.\n"
"chr2\t1\t5\tpeak3\t0\t-\n"
)
completed = run_cli(
"bed_validator.py",
"--input",
"valid.bed",
"--assembly",
"GRCh38",
"--chrom-sizes",
"chrom.sizes",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertTrue(payload["ok"])
self.assertEqual(payload["summary"]["records"], 3)
self.assertEqual(payload["contract"]["coordinate_system"], "0-based-half-open")
self.assertFalse(payload["contract"]["input_mutated"])
self.assertNotIn(str(root), completed.stdout)
def test_negative_and_out_of_bounds_intervals_fail(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / "chrom.sizes").write_text("chr1\t100\n")
(root / "invalid.bed").write_text("chr1\t-1\t10\nchr1\t90\t101\n")
completed = run_cli(
"bed_validator.py",
"--input",
"invalid.bed",
"--assembly",
"GRCh38",
"--chrom-sizes",
"chrom.sizes",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 2)
self.assertFalse(payload["ok"])
self.assertEqual(payload["errors"]["negative_coordinate"], 1)
self.assertEqual(payload["errors"]["end_beyond_contig"], 1)
def test_symlink_is_rejected(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
target = root / "target.bed"
target.write_text("chr1\t0\t10\n")
link = root / "link.bed"
try:
link.symlink_to(target)
except (OSError, NotImplementedError):
self.skipTest("symlinks unavailable")
completed = run_cli(
"bed_validator.py",
"--input",
"link.bed",
"--assembly",
"GRCh38",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 2)
self.assertFalse(payload["ok"])
self.assertIn("symlink", payload["message"].lower())
def test_url_is_rejected_without_network(self):
completed = run_cli(
"bed_validator.py",
"--input",
"https://example.invalid/private.bed",
"--assembly",
"GRCh38",
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 2)
self.assertFalse(payload["ok"])
self.assertIn("url", payload["message"].lower())
self.assertNotIn("private.bed", completed.stdout)
class CorpusAuditorTests(unittest.TestCase):
def test_detects_patient_leakage_without_emitting_values(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / "a.bed").write_text("chr1\t0\t10\n")
(root / "b.bed").write_text("chr1\t20\t30\n")
(root / "manifest.tsv").write_text(
"path\tassembly\tpatient_id\tsplit\n"
"a.bed\tGRCh38\tSECRET_PATIENT\ttrain\n"
"b.bed\tGRCh38\tSECRET_PATIENT\ttest\n"
)
completed = run_cli(
"corpus_auditor.py",
"--manifest",
"manifest.tsv",
"--group-column",
"patient_id",
"--split-column",
"split",
"--checksums",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 2)
self.assertEqual(
payload["leakage_checks"]["patient_id"]["groups_crossing_splits"],
1,
)
self.assertNotIn("SECRET_PATIENT", completed.stdout)
self.assertNotIn("a.bed", completed.stdout)
class ModelAndTokenizerTests(unittest.TestCase):
def _bundle(self, root: Path, universe_text: str = "chr1\t0\t10\nchr1\t10\t20\n"):
bundle = root / "model"
bundle.mkdir()
(bundle / "checkpoint.pt").write_bytes(
b"\x80\x04untrusted-placeholder-not-a-real-checkpoint"
)
(bundle / "config.yaml").write_text(
"vocab_size: 9\nembedding_dim: 100\npooling_method: mean\n"
)
(bundle / "universe.bed").write_text(universe_text)
return bundle
def test_model_inspector_never_deserializes_checkpoint(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
self._bundle(root)
completed = run_cli(
"model_artifact_inspector.py",
"--model-dir",
"model",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertTrue(payload["ok"])
self.assertFalse(payload["contract"]["model_deserialized"])
self.assertEqual(
payload["summary"]["risk_classes"]["deserialization"],
1,
)
self.assertNotIn("untrusted-placeholder", completed.stdout)
def test_tokenizer_plan_matches_exact_bundle_universe(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
bundle = self._bundle(root)
expected = root / "expected.bed"
expected.write_text((bundle / "universe.bed").read_text())
completed = run_cli(
"tokenizer_compatibility.py",
"--model-dir",
"model",
"--universe",
"expected.bed",
"--assembly",
"GRCh38",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertTrue(payload["ok"])
self.assertEqual(payload["artifacts"]["config"]["metadata"]["vocab_size"], 9)
self.assertTrue(all(check["passed"] for check in payload["checks"]))
def test_tokenizer_plan_rejects_universe_mismatch(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
self._bundle(root)
(root / "expected.bed").write_text("chr1\t0\t11\nchr1\t10\t20\n")
completed = run_cli(
"tokenizer_compatibility.py",
"--model-dir",
"model",
"--universe",
"expected.bed",
"--assembly",
"GRCh38",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 2)
self.assertIn(
"bundle_universe_checksum_mismatch",
payload["errors"],
)
class PlannerTests(unittest.TestCase):
def test_consensus_plan_emits_current_build_universe_command(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / "a.bed").write_text("chr1\t0\t10\n")
(root / "manifest.tsv").write_text(
"path\tassembly\n"
"a.bed\tGRCh38\n"
)
(root / "chrom.sizes").write_text("chr1\t1000\n")
(root / "coverage").mkdir()
(root / "coverage" / "all_core.bw").write_bytes(b"synthetic-bigwig")
(root / "output").mkdir()
completed = run_cli(
"consensus_plan.py",
"--manifest",
"manifest.tsv",
"--chrom-sizes",
"chrom.sizes",
"--assembly",
"GRCh38",
"--method",
"cc",
"--cutoff",
"1",
"--coverage-dir",
"coverage",
"--output-dir",
"output",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertTrue(payload["ready_to_execute"])
build_stage = next(
stage for stage in payload["stages"] if stage["stage"] == "build_universe"
)
self.assertEqual(
build_stage["argv_template"][:3],
["geniml", "build-universe", "cc"],
)
self.assertIsNone(build_stage["argv"])
self.assertFalse(payload["contract"]["commands_executed"])
def test_embedding_plan_accepts_bounded_synthetic_parquet_envelope(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / "tokens.parquet").write_bytes(b"PAR1syntheticPAR1")
(root / "universe.bed").write_text("chr1\t0\t10\n")
(root / "output").mkdir()
completed = run_cli(
"embedding_plan.py",
"--mode",
"region2vec",
"--data",
"tokens.parquet",
"--universe",
"universe.bed",
"--output-dir",
"output",
"--assembly",
"GRCh38",
cwd=root,
)
payload = json_output(completed)
self.assertEqual(completed.returncode, 0, completed.stderr)
self.assertTrue(payload["ok"])
self.assertFalse(payload["contract"]["packages_imported"])
smoke = next(
stage for stage in payload["stages"] if stage["stage"] == "api_smoke"
)
self.assertIn(
"geniml.region2vec.main.Region2VecExModel",
smoke["imports"],
)
self.assertNotIn(str(root), completed.stdout)
class MetadataFailureTests(unittest.TestCase):
def test_model_dimensions_reject_booleans(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
bundle = ModelAndTokenizerTests()._bundle(root)
(bundle / 'config.yaml').write_text('vocab_size: 9\nembedding_dim: true\n')
for name in ('model_artifact_inspector.py', 'tokenizer_compatibility.py'):
args = ['--model-dir', 'model']
if name == 'tokenizer_compatibility.py':
args += ['--universe', 'model/universe.bed', '--assembly', 'synthetic']
completed = run_cli(name, *args, cwd=root)
self.assertEqual(completed.returncode, 2, completed.stderr)
self.assertIn('config:invalid_embedding_dim', json_output(completed)['errors'])
def test_nonfinite_and_ambiguous_json_fail_cleanly(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
bundle = ModelAndTokenizerTests()._bundle(root)
for text in ('{"embedding_dim": NaN}', '{"embedding_dim": 1e999}',
'{"vocab_size":9,"vocab_size":1}', '[' * 2000 + '0' + ']' * 2000):
with self.subTest(text=text[:50]):
(bundle / 'config.json').write_text(text)
completed = run_cli('model_artifact_inspector.py', '--model-dir', 'model', cwd=root)
self.assertEqual(completed.returncode, 2, completed.stderr)
self.assertFalse(json_output(completed)['ok'])
self.assertNotIn('Traceback', completed.stderr)
def test_yaml_numeric_overflow_fails_cleanly(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
bundle = ModelAndTokenizerTests()._bundle(root)
(bundle / 'config.yaml').write_text('vocab_size: 9\nembedding_dim: 1e999\n')
completed = run_cli('model_artifact_inspector.py', '--model-dir', 'model', cwd=root)
self.assertEqual(completed.returncode, 2, completed.stderr)
self.assertFalse(json_output(completed)['ok'])
self.assertNotIn('Traceback', completed.stderr)
def test_record_limit_error_keeps_path_redacted(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
(root / 'SENSITIVE_FILENAME.bed').write_text('chr1\t0\t10\nchr1\t20\t30\n')
completed = run_cli('bed_validator.py', '--input', 'SENSITIVE_FILENAME.bed',
'--assembly', 'synthetic', '--max-records', '1', cwd=root)
self.assertEqual(completed.returncode, 2, completed.stderr)
self.assertNotIn('SENSITIVE_FILENAME', completed.stdout)
self.assertFalse(json_output(completed)['ok'])
def test_line_reader_bounds_allocation_before_checking_length(self):
import importlib.util
import io
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('geniml_safety', SCRIPTS / '_common.py')
common = importlib.util.module_from_spec(spec)
spec.loader.exec_module(common)
class BoundedReader(io.BytesIO):
def __next__(self):
raise AssertionError('unbounded line iteration')
def readline(self, size=-1):
self.assert_bound(size)
return super().readline(size)
@staticmethod
def assert_bound(size):
assert size == 11
with tempfile.TemporaryDirectory() as temporary:
path = Path(temporary) / 'large.bed'
path.write_bytes(b'x' * 100)
with patch.object(common, '_open_binary_nofollow', return_value=BoundedReader(b'x' * 100)):
with self.assertRaises(common.SafetyError):
list(common.iter_text_lines(path, max_bytes=1000, max_records=3, max_line_bytes=10))
if __name__ == "__main__":
unittest.main()