Skip to content

Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇

feat(ml): add CSIC 2010 dataset download script

CarterPerez-deva9f6d3633f19
diff --git a/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py b/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py
new file mode 100644
index 00000000..22d346f4
--- /dev/null
+++ b/PROJECTS/advanced/ai-threat-detection/backend/ml/download_csic.py
@@ -0,0 +1,111 @@
+"""
+©AngelaMos | 2026
+download_csic.py
+"""
+
+import hashlib
+import logging
+import sys
+from pathlib import Path
+from urllib.request import urlretrieve
+
+logger = logging.getLogger(__name__)
+
+DATASET_DIR = Path("data/datasets/csic2010")
+
+BASE_URL = (
+ "https://gitlab.fing.edu.uy"
+ "/gsi/web-application-attacks-datasets"
+ "/-/raw/master/csic_2010"
+)
+
+FILES = [
+ "normalTrafficTraining.txt",
+ "normalTrafficTest.txt",
+ "anomalousTrafficTest.txt",
+]
+
+MIN_FILE_BYTES = 1_000_000
+
+
+def _progress_hook(
+ block_num: int,
+ block_size: int,
+ total_size: int,
+) -> None:
+ """
+ Print download progress to stdout
+ """
+ downloaded = block_num * block_size
+ if total_size > 0:
+ pct = min(downloaded * 100 / total_size, 100)
+ sys.stdout.write(f"\r {pct:.0f}%")
+ else:
+ mb = downloaded / 1_048_576
+ sys.stdout.write(f"\r {mb:.1f} MB")
+ sys.stdout.flush()
+
+
+def _compute_sha256(path: Path) -> str:
+ """
+ Compute SHA-256 hash of a file
+ """
+ h = hashlib.sha256()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(65536), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def download_csic(
+ output_dir: Path = DATASET_DIR,
+) -> None:
+ """
+ Download CSIC 2010 dataset files
+ """
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ for filename in FILES:
+ dest = output_dir / filename
+
+ if dest.exists() and dest.stat().st_size > MIN_FILE_BYTES:
+ logger.info(
+ "Skipping %s (already exists)", filename
+ )
+ continue
+
+ url = f"{BASE_URL}/{filename}"
+ logger.info("Downloading %s", url)
+ print(f"Downloading {filename}...")
+
+ try:
+ urlretrieve(url, dest, reporthook=_progress_hook)
+ print()
+ except Exception as exc:
+ logger.error(
+ "Failed to download %s: %s",
+ filename,
+ exc,
+ )
+ print(f"\nError downloading {filename}: {exc}")
+ continue
+
+ size = dest.stat().st_size
+ sha = _compute_sha256(dest)
+ print(
+ f" Saved: {dest}"
+ f" ({size:,} bytes, sha256={sha[:12]})"
+ )
+
+ if size < MIN_FILE_BYTES:
+ print(
+ f" WARNING: {filename} is suspiciously"
+ f" small ({size:,} bytes)"
+ )
+
+ print(f"\nDataset directory: {output_dir.resolve()}")
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.INFO)
+ download_csic()
diff --git a/PROJECTS/advanced/ai-threat-detection/justfile b/PROJECTS/advanced/ai-threat-detection/justfile
index 00438df8..3a23d20d 100644
--- a/PROJECTS/advanced/ai-threat-detection/justfile
+++ b/PROJECTS/advanced/ai-threat-detection/justfile
@@ -233,6 +233,10 @@ vigil-train *ARGS:
vigil-replay *ARGS:
cd backend && uv run vigil replay {{ARGS}}
+[group('vigil')]
+vigil-download-csic:
+ cd backend && uv run python -m ml.download_csic
+
[group('vigil')]
vigil-status:
curl -sf http://localhost:8000/models/status | python -m json.tool