Skip to content

A LLM trained only on data from certain time periods to reduce modern bias

Python

Create clean_text.ipynb

Hayk Grigorian485d2cf38658
diff --git a/notebooks/clean_text.ipynb b/notebooks/clean_text.ipynb
new file mode 100644
index 0000000..36d1249
--- /dev/null
+++ b/notebooks/clean_text.ipynb
@@ -0,0 +1,35 @@
+import os
+import re
+
+RAW_DIR = "../data/raw/"
+CLEAN_DIR = "../data/cleaned/"
+
+def clean_text(text):
+ # Remove Gutenberg/Archive metadata headers & footers
+ text = re.sub(r'(?i)\*{3}.*START OF.*\*{3}', '', text)
+ text = re.sub(r'(?i)\*{3}.*END OF.*\*{3}', '', text)
+
+ # Fix line breaks and hyphenated line wraps
+ text = re.sub(r'(\w+)-\n(\w+)', r'\1\2', text)
+ text = re.sub(r'\n{2,}', '\n\n', text)
+ text = re.sub(r'\n', ' ', text)
+
+ # Remove weird characters or OCR artifacts
+ text = re.sub(r'[^\x00-\x7F]+', '', text)
+
+ return text.strip()
+
+os.makedirs(CLEAN_DIR, exist_ok=True)
+
+for filename in os.listdir(RAW_DIR):
+ if filename.endswith(".txt"):
+ with open(os.path.join(RAW_DIR, filename), "r", encoding="utf-8", errors="ignore") as f:
+ raw_text = f.read()
+
+ cleaned = clean_text(raw_text)
+
+ clean_name = filename.replace(".txt", "_cleaned.txt")
+ with open(os.path.join(CLEAN_DIR, clean_name), "w", encoding="utf-8") as f:
+ f.write(cleaned)
+
+ print(f"✅ Cleaned {filename} → {clean_name}")