A straightforward method for training your LLM, from downloading data to generating text.
"""Data preparation page: download + preprocess every dataset as background jobs."""
from __future__ import annotations
import os
import re
import sys
import time
import streamlit as st
from ui import jobs, theme
from ui.stages import DATA_DIR, DATA_SCRIPTS
theme.setup_page("Data", "📦")
theme.hero("📦 Data preparation",
"Download & preprocess the corpora: Pile (pretrain), Alpaca/Dolly/GSM8K (SFT), "
"HH-RLHF/UltraFeedback (preferences), GSM8K + arithmetic (RL).")
st.caption("Each button launches the real `scripts/prepare_*.py` as a background job. "
"Large downloads take a while, so watch the log below.")
for name, args in DATA_SCRIPTS.items():
job_id = "data_" + re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_")
s = jobs.status(job_id)
c1, c2 = st.columns([3, 1])
c1.markdown(f'<div class="stage-card" style="--c:{theme.COLORS["data"]}">'
f'<b>{name}</b> {theme.status_badge(s)}<br>'
f'<code>{" ".join(args)}</code></div>', unsafe_allow_html=True)
if c2.button("Prepare", key=f"prep_{job_id}", disabled=(s == "running")):
env_argv = [sys.executable, *args]
rec = jobs.launch(job_id, env_argv, kind="cpu")
st.session_state["last_data_job"] = job_id
st.rerun()
st.subheader("Live log")
last = st.session_state.get("last_data_job")
sel = st.selectbox("Job", [j["job_id"] for j in jobs.active_jobs() if j["job_id"].startswith("data_")] or ["(none running)"],
index=0)
job_to_show = sel if sel != "(none running)" else last
if job_to_show:
st.code(jobs.tail_log(job_to_show, 10000) or "(no log yet)", language="text")
if jobs.status(job_to_show) == "running" and st.toggle("Auto-refresh", value=True):
time.sleep(3); st.rerun()
st.subheader("Files in data/")
if os.path.isdir(DATA_DIR):
rows = []
for fn in sorted(os.listdir(DATA_DIR)):
p = os.path.join(DATA_DIR, fn)
if os.path.isfile(p):
rows.append({"file": fn, "size": f"{os.path.getsize(p)/1e6:.1f} MB"})
st.dataframe(rows, use_container_width=True, hide_index=True) if rows else st.caption("empty")
else:
st.caption("data/ does not exist yet.")