Skip to content

A LLM trained only on data from certain time periods to reduce modern bias

Python

Remove notebooks folder

Hayk Grigorianc37101bd0501
diff --git a/notebooks/.gitkeep b/notebooks/.gitkeep
deleted file mode 100644
index 8b13789..0000000
--- a/notebooks/.gitkeep
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/notebooks/clean_text.ipynb b/notebooks/clean_text.ipynb
deleted file mode 100644
index c823867..0000000
--- a/notebooks/clean_text.ipynb
+++ /dev/null
@@ -1,35 +0,0 @@
-import os
-import re
-
-RAW_DIR = "../data/raw/"
-CLEAN_DIR = "../data/cleaned/"
-
-def clean_text(text):
- # Remove Gutenberg/Archive metadata headers & footers
- text = re.sub(r'(?i)\*{3}.*START OF.*\*{3}', '', text)
- text = re.sub(r'(?i)\*{3}.*END OF.*\*{3}', '', text)
-
- # Fix line breaks and hyphenated line wraps
- text = re.sub(r'(\w+)-\n(\w+)', r'\1\2', text)
- text = re.sub(r'\n{2,}', '\n\n', text)
- text = re.sub(r'\n', ' ', text)
-
- # Remove weird characters or OCR artifacts
- text = re.sub(r'[^\x00-\x7F]+', '', text)
-
- return text.strip()
-
-os.makedirs(CLEAN_DIR, exist_ok=True)
-
-for filename in os.listdir(RAW_DIR):
- if filename.endswith(".txt"):
- with open(os.path.join(RAW_DIR, filename), "r", encoding="utf-8", errors="ignore") as f:
- raw_text = f.read()
-
- cleaned = clean_text(raw_text)
-
- clean_name = filename.replace(".txt", "_cleaned.txt")
- with open(os.path.join(CLEAN_DIR, clean_name), "w", encoding="utf-8") as f:
- f.write(cleaned)
-
- print(f" Cleaned {filename} → {clean_name}")