Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

8.9 KB
Raw Blame
#include "laya/unicodedata.hpp"

#include <cstddef>

namespace laya {
namespace unicodedata {
namespace {
#include "unicodedata_tables.inc"

template <size_t N>
bool Contains(const uint32_t (&pairs)[N], char32_t r) {
  size_t lo = 0, hi = N / 2;  // half-open over pairs
  while (lo < hi) {
    size_t mid = lo + (hi - lo) / 2;
    if (r < pairs[mid * 2]) {
      hi = mid;
    } else if (r > pairs[mid * 2 + 1]) {
      lo = mid + 1;
    } else {
      return true;
    }
  }
  return false;
}

template <size_t N>
bool Lookup(const uint64_t (&mask)[2], const uint32_t (&pairs)[N], char32_t r) {
  if (r < 0x80) return ((mask[r >> 6] >> (r & 63)) & 1) != 0;
  return Contains(pairs, r);
}

bool IsCased(char32_t r) { return IsUpper(r) || IsLower(r) || Contains(k_title, r); }

constexpr char32_t kCapitalSigma = 0x03A3, kSmallSigma = 0x03C3, kFinalSigma = 0x03C2;

bool SigmaIsFinal(std::string_view s, size_t i) {
  bool before = false;
  for (size_t j = i; j > 0;) {
    Decoded d = DecodeLastRune(s.substr(0, j));
    j -= d.width;
    if (Contains(k_case_ignorable, d.rune)) continue;
    before = Contains(k_cased, d.rune);
    break;
  }
  if (!before) return false;
  for (size_t j = i + 2; j < s.size();) {  // U+03A3 is two bytes in UTF-8
    Decoded d = DecodeRune(s.substr(j));
    j += d.width;
    if (Contains(k_case_ignorable, d.rune)) continue;
    return !Contains(k_cased, d.rune);
  }
  return true;
}

size_t InvalidPrefixLen(std::string_view b) {
  uint8_t lo = 0x80, hi = 0xBF;
  size_t need;
  uint8_t c = uint8_t(b[0]);
  if (c >= 0xC2 && c <= 0xDF) {
    need = 1;
  } else if (c == 0xE0) {
    need = 2;
    lo = 0xA0;
  } else if ((c >= 0xE1 && c <= 0xEC) || c == 0xEE || c == 0xEF) {
    need = 2;
  } else if (c == 0xED) {
    need = 2;
    hi = 0x9F;
  } else if (c == 0xF0) {
    need = 3;
    lo = 0x90;
  } else if (c >= 0xF1 && c <= 0xF3) {
    need = 3;
  } else if (c == 0xF4) {
    need = 3;
    hi = 0x8F;
  } else {
    return 1;
  }
  size_t n = 1;
  for (; n <= need && n < b.size(); n++) {
    uint8_t x = uint8_t(b[n]);
    if (x < lo || x > hi) break;
    lo = 0x80;
    hi = 0xBF;
  }
  return n;
}

}  // namespace

const char* const kVersion = kUnicodeVersion;

bool IsAlpha(char32_t r) { return Lookup(k_alpha_ascii, k_alpha, r); }
bool IsWord(char32_t r) { return Lookup(k_word_ascii, k_word, r); }
bool IsCombining(char32_t r) { return Lookup(k_combining_ascii, k_combining, r); }
bool IsDigit(char32_t r) { return Lookup(k_digit_ascii, k_digit, r); }
bool IsUpper(char32_t r) { return Lookup(k_upper_ascii, k_upper, r); }
bool IsLower(char32_t r) { return Lookup(k_lower_ascii, k_lower, r); }
bool IsSpace(char32_t r) { return Lookup(k_space_ascii, k_space, r); }
bool IsMark(char32_t r) { return Lookup(k_mark_ascii, k_mark, r); }
bool IsInitial(char32_t r) { return Lookup(k_initial_ascii, k_initial, r); }
bool IsPrintable(char32_t r) { return Lookup(k_printable_ascii, k_printable, r); }
bool IsPythonWord(char32_t r) { return r == '_' || IsWord(r) || IsDigit(r); }
bool IsWordOrDigit(char32_t r) { return IsWord(r) || IsDigit(r); }

bool IsUpperString(std::string_view s) {
  bool saw = false;
  for (size_t i = 0; i < s.size();) {
    Decoded d = DecodeRune(s.substr(i));
    i += d.width;
    if (!IsCased(d.rune)) continue;
    if (!IsUpper(d.rune)) return false;
    saw = true;
  }
  return saw;
}

Decoded DecodeRune(std::string_view s) {
  const uint8_t c = uint8_t(s[0]);
  if (c < 0x80) return {c, 1};
  size_t need;
  uint8_t lo = 0x80, hi = 0xBF;
  char32_t r;
  if (c >= 0xC2 && c <= 0xDF) {
    need = 1;
    r = c & 0x1F;
  } else if (c >= 0xE0 && c <= 0xEF) {
    need = 2;
    r = c & 0x0F;
    if (c == 0xE0) lo = 0xA0;
    if (c == 0xED) hi = 0x9F;
  } else if (c >= 0xF0 && c <= 0xF4) {
    need = 3;
    r = c & 0x07;
    if (c == 0xF0) lo = 0x90;
    if (c == 0xF4) hi = 0x8F;
  } else {
    return {kReplacement, 1};
  }
  if (s.size() < need + 1) return {kReplacement, 1};
  for (size_t i = 1; i <= need; i++) {
    uint8_t x = uint8_t(s[i]);
    if (x < lo || x > hi) return {kReplacement, 1};
    lo = 0x80;
    hi = 0xBF;
    r = (r << 6) | (x & 0x3F);
  }
  return {r, need + 1};
}

Decoded DecodeLastRune(std::string_view s) {
  const size_t end = s.size();
  if (uint8_t(s[end - 1]) < 0x80) return {uint8_t(s[end - 1]), 1};
  const size_t lim = end > 4 ? end - 4 : 0;
  size_t start = end - 1;
  while (start > lim && (uint8_t(s[start]) & 0xC0) == 0x80) start--;  // back to a start byte
  Decoded d = DecodeRune(s.substr(start, end - start));
  if (start + d.width != end) return {kReplacement, 1};
  return d;
}

bool ValidUtf8(std::string_view s) {
  for (size_t i = 0; i < s.size();) {
    if (uint8_t(s[i]) < 0x80) {
      i++;
      continue;
    }
    Decoded d = DecodeRune(s.substr(i));
    if (d.rune == kReplacement && d.width == 1) return false;
    i += d.width;
  }
  return true;
}

void AppendRune(std::string& out, char32_t r) {
  if (r > 0x10FFFF || (r >= 0xD800 && r <= 0xDFFF)) r = kReplacement;
  if (r < 0x80) {
    out.push_back(char(r));
  } else if (r < 0x800) {
    out.push_back(char(0xC0 | (r >> 6)));
    out.push_back(char(0x80 | (r & 0x3F)));
  } else if (r < 0x10000) {
    out.push_back(char(0xE0 | (r >> 12)));
    out.push_back(char(0x80 | ((r >> 6) & 0x3F)));
    out.push_back(char(0x80 | (r & 0x3F)));
  } else {
    out.push_back(char(0xF0 | (r >> 18)));
    out.push_back(char(0x80 | ((r >> 12) & 0x3F)));
    out.push_back(char(0x80 | ((r >> 6) & 0x3F)));
    out.push_back(char(0x80 | (r & 0x3F)));
  }
}

size_t RuneCount(std::string_view s) {
  size_t n = 0;
  for (size_t i = 0; i < s.size(); n++) i += DecodeRune(s.substr(i)).width;
  return n;
}

char32_t LowerRune(char32_t r) {
  if (r < 0x80) return (r >= 'A' && r <= 'Z') ? r + 32 : r;
  size_t lo = 0, hi = sizeof(k_lower_from) / sizeof(k_lower_from[0]);
  while (lo < hi) {
    size_t mid = lo + (hi - lo) / 2;
    if (r < k_lower_from[mid]) {
      hi = mid;
    } else if (r > k_lower_from[mid]) {
      lo = mid + 1;
    } else {
      return k_lower_to[mid];
    }
  }
  return r;
}

std::string Lower(std::string_view s) {
  std::string out;
  out.reserve(s.size());
  for (size_t i = 0; i < s.size();) {
    Decoded d = DecodeRune(s.substr(i));
    if (d.rune == kCapitalSigma) {
      AppendRune(out, SigmaIsFinal(s, i) ? kFinalSigma : kSmallSigma);
    } else {
      bool multi = false;
      for (const LowerMulti& m : k_lower_multi) {
        if (m.from == d.rune) {
          for (uint32_t k = 0; k < m.count; k++) AppendRune(out, m.to[k]);
          multi = true;
          break;
        }
      }
      if (!multi) AppendRune(out, LowerRune(d.rune));
    }
    i += d.width;
  }
  return out;
}

std::string Lossy(std::string_view b) {
  std::string out;
  out.reserve(b.size() + 8);
  for (size_t i = 0; i < b.size();) {
    Decoded d = DecodeRune(b.substr(i));
    if (!(d.rune == kReplacement && d.width == 1)) {
      out.append(b.substr(i, d.width));
      i += d.width;
      continue;
    }
    AppendRune(out, kReplacement);
    i += InvalidPrefixLen(b.substr(i));
  }
  return out;
}

std::string ToValid(std::string_view s) { return Lossy(s); }

std::string DecimalASCII(std::string_view s) {
  std::string out;
  for (size_t i = 0; i < s.size();) {
    Decoded d = DecodeRune(s.substr(i));
    i += d.width;
    char32_t r = d.rune;
    if (r >= 0x80 && Contains(k_digit, r)) {
      size_t lo = 0, hi = sizeof(k_digit) / sizeof(k_digit[0]) / 2 - 1;
      while (lo < hi) {  // last range starting at or before r
        size_t mid = (lo + hi + 1) / 2;
        if (k_digit[mid * 2] <= r) {
          lo = mid;
        } else {
          hi = mid - 1;
        }
      }
      r = '0' + (r - k_digit[lo * 2]) % 10;
    }
    AppendRune(out, r);
  }
  return out;
}

namespace {
std::string TrimBy(std::string_view s, bool numeric) {
  auto trims = [numeric](char32_t r) { return IsSpace(r) && !(numeric && r >= 0x1c && r <= 0x1f); };
  size_t b = 0, e = s.size();
  while (b < e) {
    Decoded d = DecodeRune(s.substr(b, e - b));
    if (!trims(d.rune)) break;
    b += d.width;
  }
  while (e > b) {
    Decoded d = DecodeLastRune(s.substr(b, e - b));
    if (!trims(d.rune)) break;
    e -= d.width;
  }
  return std::string(s.substr(b, e - b));
}
}  // namespace

std::string Strip(std::string_view s) { return TrimBy(s, false); }
std::string StripNumeric(std::string_view s) { return TrimBy(s, true); }

bool IsBlank(std::string_view s) {
  for (size_t i = 0; i < s.size();) {
    Decoded d = DecodeRune(s.substr(i));
    if (!IsSpace(d.rune)) return false;
    i += d.width;
  }
  return true;
}

std::vector<std::string> Fields(std::string_view s) {
  std::vector<std::string> out;
  bool in = false;
  size_t start = 0;
  for (size_t i = 0; i < s.size();) {
    Decoded d = DecodeRune(s.substr(i));
    if (IsSpace(d.rune)) {
      if (in) {
        out.emplace_back(s.substr(start, i - start));
        in = false;
      }
    } else if (!in) {
      in = true;
      start = i;
    }
    i += d.width;
  }
  if (in) out.emplace_back(s.substr(start));
  return out;
}

}  // namespace unicodedata
}  // namespace laya