A LLM trained only on data from certain time periods to reduce modern bias
Python
Create clean_text.ipynb
Hayk Grigorian485d2cf38658
diff --git a/notebooks/clean_text.ipynb b/notebooks/clean_text.ipynbnew file mode 100644index 0000000..36d1249--- /dev/null+++ b/notebooks/clean_text.ipynb@@ -0,0 +1,35 @@+import os+import re++RAW_DIR = "../data/raw/"+CLEAN_DIR = "../data/cleaned/"++def clean_text(text):+ # Remove Gutenberg/Archive metadata headers & footers+ text = re.sub(r'(?i)\*{3}.*START OF.*\*{3}', '', text)+ text = re.sub(r'(?i)\*{3}.*END OF.*\*{3}', '', text)++ # Fix line breaks and hyphenated line wraps+ text = re.sub(r'(\w+)-\n(\w+)', r'\1\2', text)+ text = re.sub(r'\n{2,}', '\n\n', text)+ text = re.sub(r'\n', ' ', text)++ # Remove weird characters or OCR artifacts+ text = re.sub(r'[^\x00-\x7F]+', '', text)++ return text.strip()++os.makedirs(CLEAN_DIR, exist_ok=True)++for filename in os.listdir(RAW_DIR):+ if filename.endswith(".txt"):+ with open(os.path.join(RAW_DIR, filename), "r", encoding="utf-8", errors="ignore") as f:+ raw_text = f.read()++ cleaned = clean_text(raw_text)++ clean_name = filename.replace(".txt", "_cleaned.txt")+ with open(os.path.join(CLEAN_DIR, clean_name), "w", encoding="utf-8") as f:+ f.write(cleaned)++ print(f"✅ Cleaned {filename} → {clean_name}")