Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
// Package unicodedata answers the character questions laya's Python reference answers with
// str.isalpha(), str.lower(), re's \w and friends.
//
// Go's own unicode package follows whichever Unicode release the toolchain ships, which moves
// between Go versions, so every predicate here is a table recorded from CPython (see Version)
// and detection gives the same answer on every toolchain. The tables are generated by
// scripts/gen_tables.py; nothing in this package is mutable after init, so all of it is safe for
// concurrent use.
package unicodedata
import (
"strings"
"unicode/utf8"
)
// contains reports whether r lies in one of the inclusive [lo, hi] pairs of a sorted table.
func contains(pairs []rune, r rune) bool {
lo, hi := 0, len(pairs)/2-1
for lo <= hi {
mid := int(uint(lo+hi) >> 1)
switch {
case r < pairs[mid*2]:
hi = mid - 1
case r > pairs[mid*2+1]:
lo = mid + 1
default:
return true
}
}
return false
}
// lookup answers from the ASCII bitmask when it can and from the range table otherwise.
func lookup(mask *[2]uint64, pairs []rune, r rune) bool {
if r >= 0 && r < 0x80 {
return mask[r>>6]>>(uint(r)&63)&1 != 0
}
return contains(pairs, r)
}
// IsAlpha is Python's str.isalpha() for one character.
func IsAlpha(r rune) bool { return lookup(&alphaASCII, alpha[:], r) }
// IsWord is Python's [^\W\d_]: a letter, or an Nl/No numeric, but not a decimal digit.
func IsWord(r rune) bool { return lookup(&wordASCII, word[:], r) }
// IsCombining reports a non-zero canonical combining class.
func IsCombining(r rune) bool { return lookup(&combiningASCII, combining[:], r) }
// IsDigit is Python's \d: the Nd category.
func IsDigit(r rune) bool { return lookup(&digitASCII, digit[:], r) }
// IsUpper is Python's str.isupper() for one character.
func IsUpper(r rune) bool { return lookup(&upperASCII, upper[:], r) }
// IsLower is Python's str.islower() for one character.
func IsLower(r rune) bool { return lookup(&lowerASCII, lower[:], r) }
// IsSpace is Python's str.isspace() for one character.
func IsSpace(r rune) bool { return lookup(&spaceASCII, space[:], r) }
// IsMark is Python's unicodedata.category(ch).startswith("M"): the Mn, Mc and Me categories.
func IsMark(r rune) bool { return lookup(&markASCII, mark[:], r) }
// IsInitial reports category Lu, Lt or Lo: a letter that is not lowercase.
func IsInitial(r rune) bool { return lookup(&initialASCII, initial[:], r) }
// IsPrintable is Python's str.isprintable() for one character.
func IsPrintable(r rune) bool { return lookup(&printableASCII, printable[:], r) }
// IsPythonWord is Python's \w for a str: alphanumeric or underscore.
func IsPythonWord(r rune) bool { return r == '_' || IsWord(r) || IsDigit(r) }
// IsWordOrDigit is Python's [^\W_]: a word character that is not the underscore.
func IsWordOrDigit(r rune) bool { return IsWord(r) || IsDigit(r) }
func isCased(r rune) bool {
return IsUpper(r) || IsLower(r) || contains(title[:], r)
}
// IsUpperString is Python's str.isupper() for a whole string: at least one cased character, and
// every cased character uppercase.
func IsUpperString(s string) bool {
sawCased := false
for _, r := range s {
if !isCased(r) {
continue
}
if !IsUpper(r) {
return false
}
sawCased = true
}
return sawCased
}
// DecimalASCII rewrites every Nd digit to its ASCII digit, as int() and float() read them.
// Blocks of Nd digits run zero to nine from the start of their range.
func DecimalASCII(s string) string {
var b strings.Builder
for _, r := range s {
if r >= 0x80 && contains(digit[:], r) {
lo, hi := 0, len(digit)/2-1
for lo < hi {
mid := (lo + hi + 1) / 2
if digit[mid*2] <= r {
lo = mid
} else {
hi = mid - 1
}
}
r = '0' + (r-digit[lo*2])%10
}
b.WriteRune(r)
}
return b.String()
}
// Strip is Python's str.strip() with no argument.
func Strip(s string) string {
return strings.TrimFunc(s, IsSpace)
}
// StripNumeric trims what int() and float() trim: str.isspace() characters except U+001C to
// U+001F, which str.isspace() counts and those parsers do not.
func StripNumeric(s string) string {
return strings.TrimFunc(s, func(r rune) bool { return IsSpace(r) && !(0x1c <= r && r <= 0x1f) })
}
// IsBlank is Python's `not s.strip()`.
func IsBlank(s string) bool {
for _, r := range s {
if !IsSpace(r) {
return false
}
}
return true
}
// Fields is Python's str.split() with no argument: runs of whitespace separate, and leading or
// trailing whitespace yields no empty field.
func Fields(s string) []string {
var out []string
start := -1
for i, r := range s {
if IsSpace(r) {
if start >= 0 {
out = append(out, s[start:i])
start = -1
}
} else if start < 0 {
start = i
}
}
if start >= 0 {
out = append(out, s[start:])
}
return out
}
// LowerRune is the one-character part of Python's str.lower(): the code point's lowercase when
// that is a single code point, and r otherwise. Use Lower for the full mapping.
func LowerRune(r rune) rune {
if r < 0x80 {
if 'A' <= r && r <= 'Z' {
return r + 32
}
return r
}
lo, hi := 0, len(lowerFrom)-1
for lo <= hi {
mid := int(uint(lo+hi) >> 1)
switch {
case r < lowerFrom[mid]:
hi = mid - 1
case r > lowerFrom[mid]:
lo = mid + 1
default:
return lowerTo[mid]
}
}
return r
}
const (
capitalSigma = 0x03A3
smallSigma = 0x03C3
finalSigma = 0x03C2
)
// Lower is Python's str.lower(), including the one context-dependent rule: a capital sigma
// lowercases to the final form at the end of a word and to the medial form elsewhere.
func Lower(s string) string {
ascii := true
for i := 0; i < len(s); i++ {
if c := s[i]; c >= 0x80 {
ascii = false
break
}
}
if ascii {
return strings.ToLower(s)
}
var b strings.Builder
b.Grow(len(s))
for i, r := range s {
switch {
case r == capitalSigma:
if sigmaIsFinal(s, i) {
b.WriteRune(finalSigma)
} else {
b.WriteRune(smallSigma)
}
default:
if multi, ok := lowerMulti[r]; ok {
for _, m := range multi {
b.WriteRune(m)
}
} else {
b.WriteRune(LowerRune(r))
}
}
}
return b.String()
}
// sigmaIsFinal applies the Final_Sigma condition at byte offset i of s: preceded by a cased
// letter and not followed by one, skipping case-ignorable characters on both sides.
func sigmaIsFinal(s string, i int) bool {
before := false
for j := i; j > 0; {
r, w := utf8.DecodeLastRuneInString(s[:j])
j -= w
if contains(caseIgnorable[:], r) {
continue
}
before = contains(cased[:], r)
break
}
if !before {
return false
}
for j := i + utf8.RuneLen(capitalSigma); j < len(s); {
r, w := utf8.DecodeRuneInString(s[j:])
j += w
if contains(caseIgnorable[:], r) {
continue
}
return !contains(cased[:], r)
}
return true
}
// Lossy decodes bytes replacing each maximal invalid subpart with U+FFFD, as Python's bytes.decode("utf-8", "replace") and Rust's
// String::from_utf8_lossy do: one U+FFFD for "\xe2\x82", two for "\xff\xfe".
func Lossy(b []byte) string {
if utf8.Valid(b) {
return string(b)
}
var out strings.Builder
out.Grow(len(b) + 8)
for i := 0; i < len(b); {
r, w := utf8.DecodeRune(b[i:])
if r != utf8.RuneError || w != 1 {
out.WriteRune(r)
i += w
continue
}
out.WriteRune(utf8.RuneError)
i += invalidPrefixLen(b[i:])
}
return out.String()
}
// invalidPrefixLen is the length of the longest prefix of b that could begin a valid sequence
// but is not one: at least 1.
func invalidPrefixLen(b []byte) int {
lo, hi := byte(0x80), byte(0xBF)
var need int
switch c := b[0]; {
case c >= 0xC2 && c <= 0xDF:
need = 1
case c == 0xE0:
need, lo = 2, 0xA0
case c >= 0xE1 && c <= 0xEC, c == 0xEE, c == 0xEF:
need = 2
case c == 0xED:
need, hi = 2, 0x9F
case c == 0xF0:
need, lo = 3, 0x90
case c >= 0xF1 && c <= 0xF3:
need = 3
case c == 0xF4:
need, hi = 3, 0x8F
default:
return 1
}
n := 1
for ; n <= need && n < len(b); n++ {
if b[n] < lo || b[n] > hi {
break
}
lo, hi = 0x80, 0xBF
}
return n
}
// ToValid is s with each maximal invalid subpart replaced by U+FFFD, and s itself when it is
// already valid (the common case allocates nothing). strings.ToValidUTF8 is not this: it
// collapses a run of invalid bytes into one replacement.
func ToValid(s string) string {
if utf8.ValidString(s) {
return s
}
return Lossy([]byte(s))
}