Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

8.2 KB
Raw Blame
// Package unicodedata answers the character questions laya's Python reference answers with
// str.isalpha(), str.lower(), re's \w and friends.
//
// Go's own unicode package follows whichever Unicode release the toolchain ships, which moves
// between Go versions, so every predicate here is a table recorded from CPython (see Version)
// and detection gives the same answer on every toolchain. The tables are generated by
// scripts/gen_tables.py; nothing in this package is mutable after init, so all of it is safe for
// concurrent use.
package unicodedata

import (
	"strings"
	"unicode/utf8"
)

// contains reports whether r lies in one of the inclusive [lo, hi] pairs of a sorted table.
func contains(pairs []rune, r rune) bool {
	lo, hi := 0, len(pairs)/2-1
	for lo <= hi {
		mid := int(uint(lo+hi) >> 1)
		switch {
		case r < pairs[mid*2]:
			hi = mid - 1
		case r > pairs[mid*2+1]:
			lo = mid + 1
		default:
			return true
		}
	}
	return false
}

// lookup answers from the ASCII bitmask when it can and from the range table otherwise.
func lookup(mask *[2]uint64, pairs []rune, r rune) bool {
	if r >= 0 && r < 0x80 {
		return mask[r>>6]>>(uint(r)&63)&1 != 0
	}
	return contains(pairs, r)
}

// IsAlpha is Python's str.isalpha() for one character.
func IsAlpha(r rune) bool { return lookup(&alphaASCII, alpha[:], r) }

// IsWord is Python's [^\W\d_]: a letter, or an Nl/No numeric, but not a decimal digit.
func IsWord(r rune) bool { return lookup(&wordASCII, word[:], r) }

// IsCombining reports a non-zero canonical combining class.
func IsCombining(r rune) bool { return lookup(&combiningASCII, combining[:], r) }

// IsDigit is Python's \d: the Nd category.
func IsDigit(r rune) bool { return lookup(&digitASCII, digit[:], r) }

// IsUpper is Python's str.isupper() for one character.
func IsUpper(r rune) bool { return lookup(&upperASCII, upper[:], r) }

// IsLower is Python's str.islower() for one character.
func IsLower(r rune) bool { return lookup(&lowerASCII, lower[:], r) }

// IsSpace is Python's str.isspace() for one character.
func IsSpace(r rune) bool { return lookup(&spaceASCII, space[:], r) }

// IsMark is Python's unicodedata.category(ch).startswith("M"): the Mn, Mc and Me categories.
func IsMark(r rune) bool { return lookup(&markASCII, mark[:], r) }

// IsInitial reports category Lu, Lt or Lo: a letter that is not lowercase.
func IsInitial(r rune) bool { return lookup(&initialASCII, initial[:], r) }

// IsPrintable is Python's str.isprintable() for one character.
func IsPrintable(r rune) bool { return lookup(&printableASCII, printable[:], r) }

// IsPythonWord is Python's \w for a str: alphanumeric or underscore.
func IsPythonWord(r rune) bool { return r == '_' || IsWord(r) || IsDigit(r) }

// IsWordOrDigit is Python's [^\W_]: a word character that is not the underscore.
func IsWordOrDigit(r rune) bool { return IsWord(r) || IsDigit(r) }

func isCased(r rune) bool {
	return IsUpper(r) || IsLower(r) || contains(title[:], r)
}

// IsUpperString is Python's str.isupper() for a whole string: at least one cased character, and
// every cased character uppercase.
func IsUpperString(s string) bool {
	sawCased := false
	for _, r := range s {
		if !isCased(r) {
			continue
		}
		if !IsUpper(r) {
			return false
		}
		sawCased = true
	}
	return sawCased
}

// DecimalASCII rewrites every Nd digit to its ASCII digit, as int() and float() read them.
// Blocks of Nd digits run zero to nine from the start of their range.
func DecimalASCII(s string) string {
	var b strings.Builder
	for _, r := range s {
		if r >= 0x80 && contains(digit[:], r) {
			lo, hi := 0, len(digit)/2-1
			for lo < hi {
				mid := (lo + hi + 1) / 2
				if digit[mid*2] <= r {
					lo = mid
				} else {
					hi = mid - 1
				}
			}
			r = '0' + (r-digit[lo*2])%10
		}
		b.WriteRune(r)
	}
	return b.String()
}

// Strip is Python's str.strip() with no argument.
func Strip(s string) string {
	return strings.TrimFunc(s, IsSpace)
}

// StripNumeric trims what int() and float() trim: str.isspace() characters except U+001C to
// U+001F, which str.isspace() counts and those parsers do not.
func StripNumeric(s string) string {
	return strings.TrimFunc(s, func(r rune) bool { return IsSpace(r) && !(0x1c <= r && r <= 0x1f) })
}

// IsBlank is Python's `not s.strip()`.
func IsBlank(s string) bool {
	for _, r := range s {
		if !IsSpace(r) {
			return false
		}
	}
	return true
}

// Fields is Python's str.split() with no argument: runs of whitespace separate, and leading or
// trailing whitespace yields no empty field.
func Fields(s string) []string {
	var out []string
	start := -1
	for i, r := range s {
		if IsSpace(r) {
			if start >= 0 {
				out = append(out, s[start:i])
				start = -1
			}
		} else if start < 0 {
			start = i
		}
	}
	if start >= 0 {
		out = append(out, s[start:])
	}
	return out
}

// LowerRune is the one-character part of Python's str.lower(): the code point's lowercase when
// that is a single code point, and r otherwise. Use Lower for the full mapping.
func LowerRune(r rune) rune {
	if r < 0x80 {
		if 'A' <= r && r <= 'Z' {
			return r + 32
		}
		return r
	}
	lo, hi := 0, len(lowerFrom)-1
	for lo <= hi {
		mid := int(uint(lo+hi) >> 1)
		switch {
		case r < lowerFrom[mid]:
			hi = mid - 1
		case r > lowerFrom[mid]:
			lo = mid + 1
		default:
			return lowerTo[mid]
		}
	}
	return r
}

const (
	capitalSigma = 0x03A3
	smallSigma   = 0x03C3
	finalSigma   = 0x03C2
)

// Lower is Python's str.lower(), including the one context-dependent rule: a capital sigma
// lowercases to the final form at the end of a word and to the medial form elsewhere.
func Lower(s string) string {
	ascii := true
	for i := 0; i < len(s); i++ {
		if c := s[i]; c >= 0x80 {
			ascii = false
			break
		}
	}
	if ascii {
		return strings.ToLower(s)
	}
	var b strings.Builder
	b.Grow(len(s))
	for i, r := range s {
		switch {
		case r == capitalSigma:
			if sigmaIsFinal(s, i) {
				b.WriteRune(finalSigma)
			} else {
				b.WriteRune(smallSigma)
			}
		default:
			if multi, ok := lowerMulti[r]; ok {
				for _, m := range multi {
					b.WriteRune(m)
				}
			} else {
				b.WriteRune(LowerRune(r))
			}
		}
	}
	return b.String()
}

// sigmaIsFinal applies the Final_Sigma condition at byte offset i of s: preceded by a cased
// letter and not followed by one, skipping case-ignorable characters on both sides.
func sigmaIsFinal(s string, i int) bool {
	before := false
	for j := i; j > 0; {
		r, w := utf8.DecodeLastRuneInString(s[:j])
		j -= w
		if contains(caseIgnorable[:], r) {
			continue
		}
		before = contains(cased[:], r)
		break
	}
	if !before {
		return false
	}
	for j := i + utf8.RuneLen(capitalSigma); j < len(s); {
		r, w := utf8.DecodeRuneInString(s[j:])
		j += w
		if contains(caseIgnorable[:], r) {
			continue
		}
		return !contains(cased[:], r)
	}
	return true
}

// Lossy decodes bytes replacing each maximal invalid subpart with U+FFFD, as Python's bytes.decode("utf-8", "replace") and Rust's
// String::from_utf8_lossy do: one U+FFFD for "\xe2\x82", two for "\xff\xfe".
func Lossy(b []byte) string {
	if utf8.Valid(b) {
		return string(b)
	}
	var out strings.Builder
	out.Grow(len(b) + 8)
	for i := 0; i < len(b); {
		r, w := utf8.DecodeRune(b[i:])
		if r != utf8.RuneError || w != 1 {
			out.WriteRune(r)
			i += w
			continue
		}
		out.WriteRune(utf8.RuneError)
		i += invalidPrefixLen(b[i:])
	}
	return out.String()
}

// invalidPrefixLen is the length of the longest prefix of b that could begin a valid sequence
// but is not one: at least 1.
func invalidPrefixLen(b []byte) int {
	lo, hi := byte(0x80), byte(0xBF)
	var need int
	switch c := b[0]; {
	case c >= 0xC2 && c <= 0xDF:
		need = 1
	case c == 0xE0:
		need, lo = 2, 0xA0
	case c >= 0xE1 && c <= 0xEC, c == 0xEE, c == 0xEF:
		need = 2
	case c == 0xED:
		need, hi = 2, 0x9F
	case c == 0xF0:
		need, lo = 3, 0x90
	case c >= 0xF1 && c <= 0xF3:
		need = 3
	case c == 0xF4:
		need, hi = 3, 0x8F
	default:
		return 1
	}
	n := 1
	for ; n <= need && n < len(b); n++ {
		if b[n] < lo || b[n] > hi {
			break
		}
		lo, hi = 0x80, 0xBF
	}
	return n
}

// ToValid is s with each maximal invalid subpart replaced by U+FFFD, and s itself when it is
// already valid (the common case allocates nothing). strings.ToValidUTF8 is not this: it
// collapses a run of invalid bytes into one replacement.
func ToValid(s string) string {
	if utf8.ValidString(s) {
		return s
	}
	return Lossy([]byte(s))
}