A LLM trained only on data from certain time periods to reduce modern bias
Python
Remove notebooks folder
Hayk Grigorianc37101bd0501
diff --git a/notebooks/.gitkeep b/notebooks/.gitkeepdeleted file mode 100644index 8b13789..0000000--- a/notebooks/.gitkeep+++ /dev/null@@ -1 +0,0 @@-diff --git a/notebooks/clean_text.ipynb b/notebooks/clean_text.ipynbdeleted file mode 100644index c823867..0000000--- a/notebooks/clean_text.ipynb+++ /dev/null@@ -1,35 +0,0 @@-import os-import re--RAW_DIR = "../data/raw/"-CLEAN_DIR = "../data/cleaned/"--def clean_text(text):- # Remove Gutenberg/Archive metadata headers & footers- text = re.sub(r'(?i)\*{3}.*START OF.*\*{3}', '', text)- text = re.sub(r'(?i)\*{3}.*END OF.*\*{3}', '', text)-- # Fix line breaks and hyphenated line wraps- text = re.sub(r'(\w+)-\n(\w+)', r'\1\2', text)- text = re.sub(r'\n{2,}', '\n\n', text)- text = re.sub(r'\n', ' ', text)-- # Remove weird characters or OCR artifacts- text = re.sub(r'[^\x00-\x7F]+', '', text)-- return text.strip()--os.makedirs(CLEAN_DIR, exist_ok=True)--for filename in os.listdir(RAW_DIR):- if filename.endswith(".txt"):- with open(os.path.join(RAW_DIR, filename), "r", encoding="utf-8", errors="ignore") as f:- raw_text = f.read()-- cleaned = clean_text(raw_text)-- clean_name = filename.replace(".txt", "_cleaned.txt")- with open(os.path.join(CLEAN_DIR, clean_name), "w", encoding="utf-8") as f:- f.write(cleaned)-- print(f" Cleaned {filename} → {clean_name}")