Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

11.0 KB
Raw Blame
#include <cstring>

#include "golden.hpp"
#include "laya/pyjson.hpp"
#include "laya/pyvalue.hpp"
#include "laya/sha256.hpp"
#include "laya/unicodedata.hpp"

using laya_test::At;
using laya_test::Value;
namespace ud = laya::unicodedata;
namespace pv = laya::pyvalue;

// ---- sha256: FIPS 180-4 / NIST vectors and Python hashlib goldens --------------------------------
TEST(Sha256, FipsAndHashlibVectors) {
  EXPECT_EQ(laya::Sha256Hex(""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
  EXPECT_EQ(laya::Sha256Hex("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
  EXPECT_EQ(laya::Sha256Hex("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"),
            "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
  EXPECT_EQ(laya::Sha256Hex(std::string(1000000, 'a')),
            "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");
  EXPECT_EQ(laya::Sha256Hex(std::string(55, 'a')),  // padding edge: 55, 56, 63, 64 byte messages
            "9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318");
  EXPECT_EQ(laya::Sha256Hex(std::string(56, 'a')),
            "b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a");
  EXPECT_EQ(laya::Sha256Hex(std::string(63, 'a')),
            "7d3e74a05d7db15bce4ad9ec0658ea98e3f06eeecf16b4c6fff2da457ddc2f34");
  EXPECT_EQ(laya::Sha256Hex(std::string(64, 'a')),
            "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb");
}

TEST(Sha256, IncrementalEqualsOneShot) {
  std::string data(777, 'x');
  for (size_t i = 0; i < data.size(); i++) data[i] = char(i * 31);
  laya::Sha256 h;
  for (size_t i = 0; i < data.size(); i += 13) h.Update(data.data() + i, std::min<size_t>(13, data.size() - i));
  EXPECT_EQ(laya::ToHex(h.Finish()), laya::Sha256Hex(data));
}

// ---- unicodedata: whole-Unicode digests against CPython ------------------------------------------
namespace {
std::string PredicateDigest(bool (*pred)(char32_t)) {
  std::string bits;
  for (char32_t r = 0; r < 0x110000; r++) bits.push_back(pred(r) ? '1' : '0');
  return laya::Sha256Hex(bits);
}
}  // namespace

TEST(Unicode, VersionAndPredicatesMatchCPythonEverywhere) {
  LAYA_GOLDEN_LIST(d, "lang_detect.json", "unicode_digests");
  EXPECT_STREQ(ud::kVersion, At(laya_test::Load("lang_detect.json", true), {"unicode_version"})->as_string().c_str());
  struct P { const char* name; bool (*fn)(char32_t); };
  const P preds[] = {{"alpha", ud::IsAlpha}, {"word", ud::IsWord},   {"combining", ud::IsCombining},
                     {"digit", ud::IsDigit}, {"upper", ud::IsUpper}, {"lower", ud::IsLower},
                     {"space", ud::IsSpace}, {"mark", ud::IsMark},   {"initial", ud::IsInitial}};
  for (const P& p : preds) {
    ASSERT_NE(d->Find(p.name), nullptr) << p.name;
    EXPECT_EQ(PredicateDigest(p.fn), d->Find(p.name)->as_string()) << p.name;
  }
}

TEST(Unicode, LowerMatchesCPythonEverywhere) {
  LAYA_GOLDEN_LIST(d, "lang_detect.json", "unicode_digests", "python_lower");
  laya::Sha256 h;
  for (char32_t r = 0; r < 0x110000; r++) {
    if (r >= 0xD800 && r <= 0xDFFF) continue;
    std::string one;
    ud::AppendRune(one, r);
    std::string low = ud::Lower(one), line = std::to_string(r) + ":";
    for (size_t i = 0; i < low.size();) {
      ud::Decoded x = ud::DecodeRune(std::string_view(low).substr(i));
      if (i > 0) line += ",";
      line += std::to_string(x.rune);
      i += x.width;
    }
    line += ";";
    h.Update(line);
  }
  EXPECT_EQ(laya::ToHex(h.Finish()), d->as_string());
}

TEST(Unicode, FinalSigmaAndPythonLowerRows) {
  LAYA_GOLDEN_LIST(rows, "go_extra.json", "python_lower", "cases");
  for (size_t i = 0; i < rows->size(); i++) {
    const Value& c = rows->value(i);
    EXPECT_EQ(ud::Lower(c.Find("text")->as_string()), c.Find("lower")->as_string()) << "row " << i;
  }
}

TEST(Unicode, InvalidUtf8MatchesPython) {
  LAYA_GOLDEN_LIST(fam, "go_extra.json", "invalid_utf8");
  const Value& hex = *fam->Find("hex");
  const Value& decoded = *fam->Find("decoded");
  ASSERT_EQ(hex.size(), 3000u);
  for (size_t i = 0; i < hex.size(); i++) {
    const std::string& h = hex.value(i).as_string();
    std::string bytes;
    for (size_t k = 0; k + 1 < h.size(); k += 2) bytes.push_back(char(std::stoi(h.substr(k, 2), nullptr, 16)));
    EXPECT_EQ(ud::Lossy(bytes), decoded.value(i).as_string()) << "row " << i << " " << h;
  }
}

TEST(Unicode, StripFieldsDecimal) {
  EXPECT_EQ(ud::Strip("  a b\t\xe3\x80\x80"), "a b");
  EXPECT_EQ(ud::StripNumeric("\x1f 12 "), "\x1f 12");
  EXPECT_EQ(ud::Fields(" a  b\n").size(), 2u);
  EXPECT_EQ(ud::DecimalASCII("\xd9\xa1\xd9\xa2"), "12");
  EXPECT_TRUE(ud::IsBlank(" \t"));
  EXPECT_TRUE(ud::IsUpperString("AB1"));
  EXPECT_FALSE(ud::IsUpperString("Ab"));
}

// ---- pyjson: float repr, str repr, percent, round4 -----------------------------------------------
class DoublesChunk : public ::testing::TestWithParam<int> {};
TEST_P(DoublesChunk, ReprFloatMatchesCPython) {
  LAYA_GOLDEN_LIST(rows, "python_json.json", "doubles");
  LAYA_EXPECT_ROWS(rows, 533);
  for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(rows->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
    uint64_t bits = laya_test::BitsOf(*rows->value(i).Find("bits"));
    double v;
    std::memcpy(&v, &bits, 8);
    EXPECT_EQ(laya::pyjson::ReprFloat(v), rows->value(i).Find("repr")->as_string()) << "bits " << bits;
  }
}
INSTANTIATE_TEST_SUITE_P(Golden, DoublesChunk, ::testing::ValuesIn(laya_test::Chunks(533)));

TEST(PyJson, ReprFloatNotationBoundaries) {  // fixed for exponents in [-4, 16), scientific outside
  EXPECT_EQ(laya::pyjson::ReprFloat(1e15), "1000000000000000.0");
  EXPECT_EQ(laya::pyjson::ReprFloat(1e16), "1e+16");
  EXPECT_EQ(laya::pyjson::ReprFloat(0.0001), "0.0001");
  EXPECT_EQ(laya::pyjson::ReprFloat(0.00001), "1e-05");
  EXPECT_EQ(laya::pyjson::ReprFloat(1.5e300), "1.5e+300");
  EXPECT_EQ(laya::pyjson::ReprFloat(0.30000000000000004), "0.30000000000000004");
}

TEST(PyJson, ReprStringsDigestPercentRound4) {
  LAYA_GOLDEN_LIST(rs, "python_json.json", "repr_strings");
  for (size_t i = 0; i < rs->size(); i++) {
    EXPECT_EQ(laya::pyjson::Repr(rs->value(i).value(0).as_string()), rs->value(i).value(1).as_string());
  }
  laya::Sha256 h;  // Repr("a"+chr(r)+"b") over every non-surrogate code point
  for (char32_t r = 0; r < 0x110000; r++) {
    if (r >= 0xD800 && r <= 0xDFFF) continue;
    std::string s = "a";
    ud::AppendRune(s, r);
    s += "b";
    h.Update(laya::pyjson::Repr(s));
  }
  EXPECT_EQ(laya::ToHex(h.Finish()), At(laya_test::Load("python_json.json", true), {"repr_digest"})->as_string());
  const Value* pc = At(laya_test::Load("python_json.json", true), {"percent0"});
  for (size_t i = 0; i < pc->size(); i++) {
    EXPECT_EQ(laya::pyjson::Percent0(laya_test::Num(pc->value(i).value(0))), pc->value(i).value(1).as_string());
  }
  const Value* groups = At(laya_test::Load("python_json.json", true), {"round4"});
  ASSERT_NE(groups->Find("halfway"), nullptr);
  for (size_t g = 0; g < groups->size(); g++) {
    const Value& rows = groups->value(g);
    for (size_t i = 0; i < rows.size(); i++) {
      uint64_t bits = laya_test::BitsOf(*rows.value(i).Find("bits"));
      double v;
      std::memcpy(&v, &bits, 8);
      EXPECT_EQ(laya::pyjson::Round4(v), laya_test::Num(*rows.value(i).Find("r4"))) << groups->key(g) << " " << v;
    }
  }
}

class Round4Chunk : public ::testing::TestWithParam<int> {};
TEST_P(Round4Chunk, HalfEvenMatchesPython) {
  LAYA_GOLDEN_LIST(vals, "go_extra.json", "round4", "values");
  LAYA_EXPECT_ROWS(vals, 5605);
  const Value* want = At(laya_test::Load("go_extra.json", true), {"round4", "rounded"});
  for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(vals->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
    double got = laya::pyjson::Round4(laya_test::Num(vals->value(i))), w = laya_test::Num(want->value(i));
    EXPECT_EQ(std::memcmp(&got, &w, 8), 0) << "row " << i << " " << got << " vs " << w;
  }
}
INSTANTIATE_TEST_SUITE_P(Golden, Round4Chunk, ::testing::ValuesIn(laya_test::Chunks(5605)));

// ---- pyvalue: json.loads / json.dumps ------------------------------------------------------------
class StatesChunk : public ::testing::TestWithParam<int> {};
TEST_P(StatesChunk, ParseAndDumpsMatchPython) {
  LAYA_GOLDEN_LIST(cases, "go_extra.json", "states", "cases");
  LAYA_EXPECT_ROWS(cases, 2000);
  for (size_t i = size_t(GetParam()) * laya_test::kChunk; i < std::min(cases->size(), (size_t(GetParam()) + 1) * laya_test::kChunk); i++) {
    const std::string& text = cases->value(i).Find("text")->as_string();
    auto v = pv::Parse(text);
    ASSERT_TRUE(v.ok()) << i << " " << v.error();
    std::string got;
    if (v.value().kind() == pv::Value::Kind::String) {
      got = v.value().as_string();
    } else {
      auto d = pv::Dumps(v.value());
      ASSERT_TRUE(d.ok()) << i << " " << d.error();
      got = d.value();
    }
    EXPECT_EQ(got, cases->value(i).Find("serialized")->as_string()) << "row " << i;
    EXPECT_EQ(v.value().kind() == pv::Value::Kind::Array, cases->value(i).Find("list")->as_bool()) << i;
  }
}
INSTANTIATE_TEST_SUITE_P(Golden, StatesChunk, ::testing::ValuesIn(laya_test::Chunks(2000)));

TEST(PyValue, PythonRejectsAndLoneSurrogates) {
  LAYA_GOLDEN_LIST(bad, "go_extra.json", "states", "python_rejects");
  for (size_t i = 0; i < bad->size(); i++) EXPECT_FALSE(pv::Parse(bad->value(i).as_string()).ok()) << bad->value(i).as_string().substr(0, 40);
  const Value* lone = At(laya_test::Load("go_extra.json", true), {"states", "go_refuses"});
  for (size_t i = 0; i < lone->size(); i++) {
    auto r = pv::Parse(lone->value(i).as_string());
    ASSERT_FALSE(r.ok());
    EXPECT_NE(r.error().find("lone surrogate"), std::string::npos) << r.error();
  }
}

TEST(PyValue, DuplicateKeysNaNDepthBigIntRepr) {
  auto d = pv::Parse("{\"a\": 1, \"b\": 2, \"a\": 3}");
  ASSERT_TRUE(d.ok());
  EXPECT_EQ(pv::Dumps(d.value()).value(), "{\"a\": 3, \"b\": 2}");  // first position, last value
  auto n = pv::Parse("[NaN, Infinity, -Infinity, 2, 2.0, 1e400, -0, 12345678901234567890123]");
  ASSERT_TRUE(n.ok());
  EXPECT_EQ(pv::Dumps(n.value()).value(), "[NaN, Infinity, -Infinity, 2, 2.0, Infinity, 0, 12345678901234567890123]");
  EXPECT_EQ(pv::Repr(n.value()), "[nan, inf, -inf, 2, 2.0, inf, 0, 12345678901234567890123]");
  std::string deep(600, '[');
  deep += std::string(600, ']');
  EXPECT_FALSE(pv::Parse(deep).ok());
  // The outermost value is depth 0, so kMaxDepth + 1 brackets parse and kMaxDepth + 2 do not.
  std::string ok(pv::kMaxDepth + 1, '[');
  ok += std::string(pv::kMaxDepth + 1, ']');
  EXPECT_TRUE(pv::Parse(ok).ok());
  std::string over(pv::kMaxDepth + 2, '[');
  over += std::string(pv::kMaxDepth + 2, ']');
  EXPECT_FALSE(pv::Parse(over).ok());
  EXPECT_FALSE(pv::Parse(std::string(4301, '9')).ok());
  EXPECT_TRUE(pv::Parse(std::string(4300, '9')).ok());
  EXPECT_FALSE(pv::Parse("\"\\ud800\"").ok());
  EXPECT_FALSE(pv::Parse("\xff").ok());
  auto o = pv::Parse("{\"k\": [\"it's\", null, true]}");
  EXPECT_EQ(pv::Repr(o.value()), "{'k': [\"it's\", None, True]}");
  EXPECT_EQ(pv::TypeName(o.value()), "dict");
}