Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company đŸ‘‡
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
feat(ml): add training orchestrator with MLflow and validation
CarterPerez-deva26d68e8d1cc
diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.pynew file mode 100644index 00000000..3b423a80--- /dev/null+++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/orchestrator.py@@ -0,0 +1,253 @@+"""+©AngelaMos | 2026+orchestrator.py+"""++import json+import logging+from dataclasses import dataclass+from pathlib import Path++import numpy as np++from ml.experiment import VigilExperiment+from ml.export_onnx import (+ export_autoencoder,+ export_isolation_forest,+ export_random_forest,+)+from ml.splitting import prepare_training_data+from ml.train_autoencoder import train_autoencoder+from ml.train_classifiers import (+ train_isolation_forest,+ train_random_forest,+)+from ml.validation import ValidationResult, validate_ensemble++logger = logging.getLogger(__name__)++N_FEATURES = 35++AE_FILENAME = "ae.onnx"+RF_FILENAME = "rf.onnx"+IF_FILENAME = "if.onnx"+SCALER_FILENAME = "scaler.json"+THRESHOLD_FILENAME = "threshold.json"+++@dataclass+class TrainingResult:+ """+ Aggregated results from a full training pipeline run+ """++ ae_metrics: dict[str, float]+ rf_metrics: dict[str, float]+ if_metrics: dict[str, float]+ ensemble_metrics: ValidationResult | None+ passed_gates: bool+ output_dir: Path+ mlflow_run_id: str | None+++class TrainingOrchestrator:+ """+ End-to-end training pipeline that splits data, trains all 3 models,+ exports to ONNX, validates the ensemble, and logs to MLflow+ """++ def __init__(+ self,+ output_dir: Path,+ experiment_name: str = "angelusvigil-training",+ epochs: int = 100,+ batch_size: int = 256,+ ) -> None:+ self._output_dir = output_dir+ self._experiment_name = experiment_name+ self._epochs = epochs+ self._batch_size = batch_size++ def run(+ self,+ X: np.ndarray,+ y: np.ndarray,+ ) -> TrainingResult:+ """+ Execute the full training pipeline+ """+ self._output_dir.mkdir(parents=True, exist_ok=True)++ split = prepare_training_data(X, y)++ logger.info(+ "Split: train=%d val=%d test=%d normal_train=%d",+ len(split.X_train),+ len(split.X_val),+ len(split.X_test),+ len(split.X_normal_train),+ )++ with VigilExperiment(+ self._experiment_name+ ) as experiment:+ experiment.log_params({+ "epochs": self._epochs,+ "batch_size": self._batch_size,+ "n_samples": len(X),+ "n_attack": int(np.sum(y == 1)),+ "n_normal": int(np.sum(y == 0)),+ "n_features": X.shape[1],+ })++ ae_result = self._train_ae(+ split.X_normal_train+ )+ ae_metrics = {+ "ae_threshold": ae_result["threshold"],+ "ae_final_train_loss": ae_result[+ "history"+ ]["train_loss"][-1],+ "ae_final_val_loss": ae_result[+ "history"+ ]["val_loss"][-1],+ }++ rf_result = self._train_rf(+ split.X_train, split.y_train+ )+ rf_metrics = rf_result["metrics"]++ if_result = self._train_if(+ split.X_normal_train+ )+ if_metrics = if_result["metrics"]++ self._export_models(+ ae_result, rf_result, if_result+ )++ experiment.log_metrics(ae_metrics)+ experiment.log_metrics(+ {f"rf_{k}": v for k, v in rf_metrics.items()}+ )++ try:+ ensemble = validate_ensemble(+ self._output_dir,+ split.X_test,+ split.y_test,+ )+ experiment.log_metrics({+ "ensemble_precision": ensemble.precision,+ "ensemble_recall": ensemble.recall,+ "ensemble_f1": ensemble.f1,+ "ensemble_pr_auc": ensemble.pr_auc,+ "ensemble_roc_auc": ensemble.roc_auc,+ })+ passed = ensemble.passed_gates+ except Exception:+ logger.exception("Ensemble validation failed")+ ensemble = None+ passed = False++ for artifact in self._output_dir.iterdir():+ experiment.log_artifact(artifact)++ run_id = experiment.run_id++ logger.info(+ "Training complete: passed_gates=%s run_id=%s",+ passed,+ run_id,+ )++ return TrainingResult(+ ae_metrics=ae_metrics,+ rf_metrics=rf_metrics,+ if_metrics=if_metrics,+ ensemble_metrics=ensemble,+ passed_gates=passed,+ output_dir=self._output_dir,+ mlflow_run_id=run_id,+ )++ def _train_ae(+ self, X_normal: np.ndarray+ ) -> dict:+ """+ Train the autoencoder on normal-only data+ """+ logger.info(+ "Training autoencoder (%d epochs, %d samples)",+ self._epochs,+ len(X_normal),+ )+ return train_autoencoder(+ X_normal,+ epochs=self._epochs,+ batch_size=self._batch_size,+ )++ def _train_rf(+ self, X: np.ndarray, y: np.ndarray+ ) -> dict:+ """+ Train the random forest classifier+ """+ logger.info(+ "Training random forest (%d samples)",+ len(X),+ )+ return train_random_forest(X, y)++ def _train_if(+ self, X_normal: np.ndarray+ ) -> dict:+ """+ Train the isolation forest on normal-only data+ """+ logger.info(+ "Training isolation forest (%d samples)",+ len(X_normal),+ )+ return train_isolation_forest(X_normal)++ def _export_models(+ self,+ ae_result: dict,+ rf_result: dict,+ if_result: dict,+ ) -> None:+ """+ Export all 3 models to ONNX and save scaler/threshold+ """+ export_autoencoder(+ ae_result["model"],+ self._output_dir / AE_FILENAME,+ )+ ae_result["scaler"].save_json(+ self._output_dir / SCALER_FILENAME+ )+ threshold_data = {+ "threshold": ae_result["threshold"]+ }+ (self._output_dir / THRESHOLD_FILENAME).write_text(+ json.dumps(threshold_data, indent=2)+ )++ export_random_forest(+ rf_result["model"],+ N_FEATURES,+ self._output_dir / RF_FILENAME,+ )++ export_isolation_forest(+ if_result["model"],+ N_FEATURES,+ self._output_dir / IF_FILENAME,+ )++ logger.info(+ "Exported models to %s", self._output_dir+ )diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.pyindex 89c91a91..d42e21c4 100644--- a/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py+++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/splitting.py@@ -64,8 +64,13 @@ def prepare_training_data(class_counts = np.bincount(y_train)minority_count = class_counts.min()+ majority_count = class_counts.max()+ current_ratio = minority_count / majority_count- if minority_count >= smote_k + 1:+ if (+ minority_count >= smote_k + 1+ and current_ratio < smote_strategy+ ):sampler = SMOTE(sampling_strategy=smote_strategy,k_neighbors=smote_k,diff --git a/PROJECTS/advanced/ai-threat-detection/backend/tests/test_orchestrator.py b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_orchestrator.pynew file mode 100644index 00000000..8d2c76d0--- /dev/null+++ b/PROJECTS/advanced/ai-threat-detection/backend/tests/test_orchestrator.py@@ -0,0 +1,180 @@+"""+©AngelaMos | 2026+test_orchestrator.py+"""++import json+from pathlib import Path++import numpy as np++from ml.orchestrator import TrainingOrchestrator, TrainingResult++N_FEATURES = 35+EXPECTED_FILES = [+ "ae.onnx",+ "rf.onnx",+ "if.onnx",+ "scaler.json",+ "threshold.json",+]+++def _make_dataset() -> tuple[np.ndarray, np.ndarray]:+ """+ Generate a small synthetic dataset for testing+ """+ rng = np.random.default_rng(42)+ X_normal = rng.standard_normal(+ (200, N_FEATURES)+ ).astype(np.float32)+ X_attack = (+ rng.standard_normal(+ (80, N_FEATURES)+ ).astype(np.float32)+ + 2.0+ )+ X = np.vstack([X_normal, X_attack])+ y = np.array(+ [0] * 200 + [1] * 80, dtype=np.int32+ )+ return X, y+++class TestTrainingOrchestrator:+ """+ Test the end-to-end training orchestrator+ """++ def test_produces_all_output_files(+ self, tmp_path: Path+ ) -> None:+ """+ Orchestrator produces all 5 expected output files+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ orch.run(X, y)++ for filename in EXPECTED_FILES:+ assert (+ tmp_path / filename+ ).exists(), f"Missing {filename}"++ def test_returns_training_result(+ self, tmp_path: Path+ ) -> None:+ """+ Returns a TrainingResult dataclass+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ result = orch.run(X, y)++ assert isinstance(result, TrainingResult)++ def test_scaler_json_has_required_keys(+ self, tmp_path: Path+ ) -> None:+ """+ scaler.json contains center, scale, and n_features+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ orch.run(X, y)++ scaler_data = json.loads(+ (tmp_path / "scaler.json").read_text()+ )+ assert "center" in scaler_data+ assert "scale" in scaler_data+ assert "n_features" in scaler_data++ def test_threshold_json_has_float(+ self, tmp_path: Path+ ) -> None:+ """+ threshold.json contains a float threshold value+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ orch.run(X, y)++ threshold_data = json.loads(+ (tmp_path / "threshold.json").read_text()+ )+ assert "threshold" in threshold_data+ assert isinstance(+ threshold_data["threshold"], float+ )++ def test_result_has_per_model_metrics(+ self, tmp_path: Path+ ) -> None:+ """+ TrainingResult includes metrics for each model+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ result = orch.run(X, y)++ assert "ae_threshold" in result.ae_metrics+ assert "f1" in result.rf_metrics+ assert "n_samples" in result.if_metrics++ def test_ensemble_metrics_present(+ self, tmp_path: Path+ ) -> None:+ """+ Ensemble validation metrics are populated+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ result = orch.run(X, y)++ assert result.ensemble_metrics is not None+ assert 0.0 <= result.ensemble_metrics.f1 <= 1.0++ def test_mlflow_run_id_set(+ self, tmp_path: Path+ ) -> None:+ """+ MLflow run ID is captured in the result+ """+ X, y = _make_dataset()+ mlflow_dir = tmp_path / "mlruns"+ mlflow_dir.mkdir()+ orch = TrainingOrchestrator(+ output_dir=tmp_path / "models",+ epochs=3,+ )+ result = orch.run(X, y)++ assert result.mlflow_run_id is not None+ assert len(result.mlflow_run_id) == 32++ def test_passed_gates_is_bool(+ self, tmp_path: Path+ ) -> None:+ """+ passed_gates is a boolean value+ """+ X, y = _make_dataset()+ orch = TrainingOrchestrator(+ output_dir=tmp_path, epochs=3+ )+ result = orch.run(X, y)++ assert isinstance(result.passed_gates, bool)