Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Shared text-extraction helpers for v0.55.0 eval-design + canary-discovery.
Extracted out of `eval_design.py` so `canary_discovery.py` no longer
imports private symbols across modules (code-review LOW fix — removes
hidden coupling).
Pure functions — no I/O, no torch.
"""
from __future__ import annotations
import unicodedata
from collections.abc import Mapping, Sequence
from typing import List
# Small stop-words list used by the TF-IDF salience clustering; just
# enough to filter the dominant function-word tokens.
STOPWORDS = frozenset(
{
"a", "an", "and", "are", "as", "at", "be", "but", "by", "do", "for",
"from", "have", "i", "in", "is", "it", "of", "on", "or", "that", "the",
"this", "to", "was", "were", "will", "with", "you",
}
)
def _is_no_space(char: str) -> bool:
cp = ord(char)
return (
# CJK Unified and Extensions A/B
0x4E00 <= cp <= 0x9FFF or
0x3400 <= cp <= 0x4DBF or
0x20000 <= cp <= 0x2A6DF or
# CJK Extension C onward (covers Ext C through Ext H)
0x2A700 <= cp <= 0x323AF or
# Kana and Halfwidth Katakana
0x3040 <= cp <= 0x309F or
0x30A0 <= cp <= 0x30FF or
0xFF65 <= cp <= 0xFF9F or
# Southeast Asian scripts
0x0E00 <= cp <= 0x0E7F or # Thai
0x0E80 <= cp <= 0x0EFF or # Lao
0x1000 <= cp <= 0x109F or # Myanmar
0x1780 <= cp <= 0x17FF # Khmer
)
def row_text(row: Mapping[str, object]) -> str:
"""Best-effort text extraction from the *output* side of a dataset row.
Soup datasets normalise to ``{"messages": [...]}``; this helper
pulls the assistant turn(s) when present, falling back to common
SFT fields. Returns an empty string on missing data.
"""
if not isinstance(row, Mapping):
return ""
messages = row.get("messages")
if isinstance(messages, Sequence) and not isinstance(messages, (str, bytes)):
chunks: List[str] = []
for msg in messages:
if not isinstance(msg, Mapping):
continue
if msg.get("role") == "assistant":
content = msg.get("content")
if isinstance(content, str):
chunks.append(content)
if chunks:
return "\n".join(chunks)
for key in ("output", "completion", "chosen", "response", "answer", "text"):
val = row.get(key)
if isinstance(val, str) and val:
return val
return ""
_MAX_MARK_RUN = 30 # UAX #15 stream-safe bound; NFC reordering is quadratic in a mark run
def _stream_safe(text: str) -> str:
out: List[str] = []
run = 0
for ch in text:
if unicodedata.category(ch)[0] == "M":
run += 1
if run > _MAX_MARK_RUN:
continue
else:
run = 0
out.append(ch)
return "".join(out)
def tokenize(text: str, *, filter_stopwords: bool = True) -> List[str]:
"""Tokenise to lowercase word/subword runs across Latin and non-Latin scripts.
Normalises Unicode (NFC, lowercase, dotless/dotted I handling), supports
non-Latin alphabetic scripts (Cyrillic, Greek, Arabic, Hebrew, Devanagari,
etc.), non-spaced scripts (CJK, Kana, Thai via character bigrams), and
digits. When ``filter_stopwords=True``, filters English stopwords and short
ASCII words <= 2 characters.
Returns an empty list for empty / non-string input.
"""
if not text or not isinstance(text, str):
return []
norm = (
unicodedata.normalize("NFC", _stream_safe(text))
.replace("İ", "i")
.lower()
.replace("\u0307", "")
)
raw_tokens: List[str] = []
current: List[str] = []
for ch in norm:
cat = unicodedata.category(ch)
if cat[0] in ("L", "M", "N") or ch in ("_", "-"):
current.append(ch)
else:
if current:
raw_tokens.append("".join(current))
current = []
if current:
raw_tokens.append("".join(current))
result: List[str] = []
for tok in raw_tokens:
clean = tok.strip("-_")
if not clean or all(unicodedata.category(c)[0] == "M" for c in clean):
continue
if any(_is_no_space(c) for c in clean):
if len(clean) == 1:
result.append(clean)
else:
for i in range(len(clean) - 1):
result.append(clean[i : i + 2])
else:
if filter_stopwords:
if clean in STOPWORDS:
continue
if clean.isascii() and clean.isalpha() and len(clean) <= 2:
continue
result.append(clean)
return result