Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

33.1 KB
Raw Blame
// Tests of the ONNX backend. The pure checks run anywhere; the tests that load a graph need
// LAYA_ONNX_GRAPH (multilingual), LAYA_ONNX_GRAPH_ENGLISH and LAYA_CHECKPOINTS, and with
// LAYA_REQUIRE_GOLDENS=1 (the onnx lane) a missing one FAILS instead of skipping.
#include <gtest/gtest.h>

#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <limits>
#include <map>
#include <mutex>
#include <set>
#include <string>
#include <thread>
#include <vector>

#include <sys/wait.h>
#include <unistd.h>

extern char** environ;

#include "golden.hpp"
#include "laya/onnx.hpp"
#include "laya/onnx_graph.hpp"

namespace {

namespace fs = std::filesystem;
using laya::Result;
using laya::infer::Batch;
using laya::onnx::OutTensor;
using laya::onnx::TensorInfo;
using laya::pyvalue::Value;
using Kind = Value::Kind;

// ---------------------------------------------------------------- graph checks (no runtime needed)

std::vector<TensorInfo> GoodInputs() {
  return {{"input_ids", "int64", {-1, -1}},
          {"attention_mask", "int64", {-1, -1}},
          {"marker_pos", "int64", {-1, -1}},
          {"marker_mask", "bool", {-1, -1}},
          {"qtype", "int64", {-1}}};
}
std::vector<TensorInfo> GoodOutputs() { return {{"logits", "float32", {-1, -1}}, {"act_logits", "float32", {-1, 2}}}; }

TEST(OnnxGraph, TheExportedShapeIsAccepted) { EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), GoodOutputs()), ""); }

TEST(OnnxGraph, InputOrderInTheGraphDoesNotMatter) {
  auto in = GoodInputs();
  std::swap(in[0], in[4]);
  std::swap(in[1], in[3]);
  EXPECT_EQ(laya::onnx::CheckGraph(in, GoodOutputs()), "");
}

TEST(OnnxGraph, EveryInputIsHeldToItsName) {
  for (size_t i = 0; i < 5; i++) {
    auto in = GoodInputs();
    in[i].name = "renamed";
    const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());
    EXPECT_NE(e.find("missing input"), std::string::npos) << i << ": " << e;
  }
}

TEST(OnnxGraph, EveryInputIsHeldToItsDtype) {
  for (size_t i = 0; i < 5; i++) {
    auto in = GoodInputs();
    in[i].dtype = in[i].dtype == "int64" ? "float32" : "int64";
    const std::string e = laya::onnx::CheckGraph(in, GoodOutputs());
    EXPECT_NE(e.find(in[i].name), std::string::npos) << e;
    EXPECT_NE(e.find("want"), std::string::npos) << e;
  }
}

TEST(OnnxGraph, TheMaskMustBeBoolNotInt64) {
  auto in = GoodInputs();
  in[3].dtype = "int64";
  EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("is int64, want bool"), std::string::npos);
}

TEST(OnnxGraph, RankIsChecked) {
  auto in = GoodInputs();
  in[4].dims = {-1, -1};
  EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 2, want 1"), std::string::npos);
  in = GoodInputs();
  in[0].dims = {-1};
  EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("rank 1, want 2"), std::string::npos);
}

TEST(OnnxGraph, AFixedBatchDimensionIsRefused) {
  auto out = GoodOutputs();
  out[1].dims = {1, 2};  // the layout of onnx/full-multilingual: act_logits traced at batch 1
  EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed batch dimension 1"), std::string::npos);
  auto in = GoodInputs();
  in[0].dims = {4, -1};
  EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("fixed batch dimension 4"), std::string::npos);
}

TEST(OnnxGraph, ActLogitsHasExactlyTwoColumns) {
  auto out = GoodOutputs();
  out[1].dims = {-1, 3};
  EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("fixed size 3"), std::string::npos);
  out[1].dims = {-1, -1};
  EXPECT_EQ(laya::onnx::CheckGraph(GoodInputs(), out), "");
}

TEST(OnnxGraph, ExtrasAndDuplicatesAreRefused) {
  auto in = GoodInputs();
  in.push_back({"extra", "int64", {-1}});
  EXPECT_NE(laya::onnx::CheckGraph(in, GoodOutputs()).find("unexpected input \"extra\""), std::string::npos);
  auto out = GoodOutputs();
  out.push_back({"logits", "float32", {-1, -1}});
  EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("duplicate output"), std::string::npos);
  EXPECT_NE(laya::onnx::CheckGraph({}, GoodOutputs()).find("missing input"), std::string::npos);
}

TEST(OnnxGraph, AnOutputDtypeIsChecked) {
  auto out = GoodOutputs();
  out[0].dtype = "int64";
  EXPECT_NE(laya::onnx::CheckGraph(GoodInputs(), out).find("output \"logits\" is int64"), std::string::npos);
}

// ---------------------------------------------------------------- paths

class TempDir {
 public:
  TempDir() {
    static std::atomic<int> n{0};
    path_ = fs::temp_directory_path() / ("laya_onnx_test_" + std::to_string(::getpid()) + "_" + std::to_string(n++));
    fs::create_directories(path_);
  }
  ~TempDir() {
    std::error_code ec;
    fs::remove_all(path_, ec);
  }
  TempDir(const TempDir&) = delete;
  TempDir& operator=(const TempDir&) = delete;
  std::string File(const std::string& name, const std::string& body = "x") const {
    std::ofstream(path_ / name, std::ios::binary) << body;
    return (path_ / name).string();
  }
  std::string str() const { return path_.string(); }

 private:
  fs::path path_;
};

TEST(OnnxPath, MissingAndDirectoryAndEmptyAreErrors) {
  TempDir d;
  EXPECT_NE(laya::onnx::CheckPath(d.str() + "/nope.onnx").find("no such file"), std::string::npos);
  EXPECT_NE(laya::onnx::CheckPath(d.str()).find("is a directory"), std::string::npos);
  EXPECT_NE(laya::onnx::CheckPath("").find("empty"), std::string::npos);
  EXPECT_EQ(laya::onnx::CheckPath(d.File("laya.onnx")), "");
}

TEST(OnnxPath, TheSplitLayoutIsRefusedWhicheverPathIsGiven) {
  TempDir d;
  const std::string enc = d.File("encoder.onnx");
  const std::string head = d.File("head.onnx");
  for (const std::string& p : {enc, head, d.str(), d.str() + "/laya.onnx"}) {
    const std::string e = laya::onnx::CheckPath(p);
    EXPECT_NE(e.find("split encoder.onnx + head.onnx layout is not supported"), std::string::npos) << p << ": " << e;
    EXPECT_NE(e.find("export_onnx.py"), std::string::npos) << e;
  }
}

TEST(OnnxPath, OneHalfAloneIsNotTheSplitLayout) {
  TempDir d;
  const std::string enc = d.File("encoder.onnx");
  EXPECT_EQ(laya::onnx::CheckPath(enc), "");
}

// ---------------------------------------------------------------- collecting outputs

Batch Tiny(int64_t rows, int64_t markers) {
  Batch b;
  b.rows = rows;
  b.length = 3;
  b.markers = markers;
  b.input_ids.assign(size_t(rows * 3), 1);
  b.attention_mask.assign(size_t(rows * 3), 1);
  b.marker_pos.assign(size_t(rows * markers), 1);
  b.marker_mask.assign(size_t(rows * markers), 1);
  b.qtype.assign(size_t(rows), 0);
  return b;
}

OutTensor Tensor(const std::vector<float>& v, std::vector<int64_t> shape) {
  OutTensor t;
  t.shape = std::move(shape);
  t.data = v.data();
  t.count = int64_t(v.size());
  return t;
}

TEST(OnnxCollect, ACorrectPairIsCopied) {
  const Batch b = Tiny(2, 3);
  const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};
  auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));
  ASSERT_TRUE(r.ok()) << r.error();
  EXPECT_EQ(r.value().logits, l);
  EXPECT_EQ(r.value().act_logits, a);
}

TEST(OnnxCollect, ASlicedOutputIsTheOutputOfTheRightRows) {
  const Batch b = Tiny(2, 3);
  const std::vector<float> l = {1, 2, 3, 4, 5, 6}, a = {7, 8, 9, 10};
  auto r = laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {2, 2}));
  ASSERT_TRUE(r.ok());
  EXPECT_EQ(r.value().logits[3], 4.0f);  // row 1, column 0
  EXPECT_EQ(r.value().act_logits[2], 9.0f);
}

TEST(OnnxCollect, ANaNIsAnErrorNotAnAnswer) {
  const Batch b = Tiny(1, 3);
  std::vector<float> l = {1, std::numeric_limits<float>::quiet_NaN(), 3}, a = {1, 2};
  auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("NaN"), std::string::npos);
  l[1] = 0;
  a[0] = std::numeric_limits<float>::quiet_NaN();
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());
}

TEST(OnnxCollect, AnInfinityIsAnErrorNotAnAnswer) {
  const Batch b = Tiny(1, 3);
  const float inf = std::numeric_limits<float>::infinity();
  for (float bad : {inf, -inf}) {
    std::vector<float> l = {1, bad, 3}, a = {1, 2};
    auto r = laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2}));
    ASSERT_FALSE(r.ok());
    EXPECT_NE(r.error().find("Inf"), std::string::npos);
    l[1] = 0;
    a[1] = bad;
    EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {1, 3}), Tensor(a, {1, 2})).ok());
  }
}

TEST(OnnxCollect, WrongShapesAreErrors) {
  const Batch b = Tiny(2, 3);
  const std::vector<float> l(6, 1.f), a(4, 1.f), small(2, 1.f);
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {3, 2}), Tensor(a, {2, 2})).ok());
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3}), Tensor(a, {4, 1})).ok());
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {6}), Tensor(a, {2, 2})).ok());
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(l, {2, 3, 1}), Tensor(a, {2, 2})).ok());
  EXPECT_FALSE(laya::onnx::CollectOutput(b, Tensor(small, {2, 3}), Tensor(a, {2, 2})).ok());  // shape says 6, holds 2
}

// ---------------------------------------------------------------- options

TEST(OnnxOpen, ThreadCountsAreRange_Checked) {
  laya::onnx::Options o;
  o.intra_op_threads = -1;
  EXPECT_FALSE(laya::onnx::Open("/nonexistent/laya.onnx", o).ok());
  o.intra_op_threads = laya::onnx::kMaxThreads + 1;
  const auto r = laya::onnx::Open("/nonexistent/laya.onnx", o);
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("intra_op_threads"), std::string::npos);
}

TEST(OnnxOpen, AMissingGraphIsAnError) {
  const auto r = laya::onnx::Open("/nonexistent/dir/laya.onnx");
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("no such file"), std::string::npos);
}

TEST(OnnxOpen, AGarbageFileIsAnErrorNotAnException) {
  TempDir d;
  const auto r = laya::onnx::Open(d.File("laya.onnx", "this is not a protobuf \x01\x02\xff"));
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
}

TEST(OnnxOpen, AnEmptyFileIsAnError) {
  TempDir d;
  EXPECT_FALSE(laya::onnx::Open(d.File("laya.onnx", "")).ok());
}

// ONNX Runtime refuses external data that leaves the graph's directory (`..` or an absolute path);
// this pins that, with a real file sitting where the graph points.
#include "onnx_hostile_graphs.inc"
TEST(OnnxOpen, ExternalDataOutsideTheGraphDirectoryIsRefused) {
  TempDir d;
  fs::create_directories(d.str() + "/sub");
  {
    std::ofstream(d.str() + "/secret.bin", std::ios::binary) << std::string(8, '\0');
  }
  const struct {
    const char* name;
    const char* bytes;
    size_t size;
  } cases[] = {{"dotdot", kDotdotGraph, kDotdotGraphSize}, {"absolute", kAbsGraph, kAbsGraphSize}};
  for (const auto& c : cases) {
    const std::string path = d.str() + "/sub/" + c.name + ".onnx";
    std::ofstream(path, std::ios::binary).write(c.bytes, std::streamsize(c.size));
    const auto r = laya::onnx::Open(path);
    ASSERT_FALSE(r.ok()) << c.name << " opened";
    EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
  }
}

TEST(OnnxOpen, TheSplitLayoutIsRefusedByOpen) {
  TempDir d;
  d.File("encoder.onnx");
  d.File("head.onnx");
  const auto r = laya::onnx::Open(d.str() + "/encoder.onnx");
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("split encoder.onnx + head.onnx layout"), std::string::npos) << r.error();
}

TEST(OnnxRuntime, TheLoadedLibraryIsThePinnedVersion) { EXPECT_EQ(laya::onnx::RuntimeVersion(), LAYA_ORT_VERSION); }

// ---------------------------------------------------------------- real graphs

struct Rig {
  const char* checkpoint;    // directory name under LAYA_CHECKPOINTS
  const char* graph_env;     // names the exported laya.onnx
  const char* golden;        // predict golden recorded from that graph
};
const Rig kRigs[] = {{"multilingual", "LAYA_ONNX_GRAPH", "predict.json"},
                     {"english", "LAYA_ONNX_GRAPH_ENGLISH", "predict-english.json"}};

// "" and a recorded failure (or skip) when the rig is not there.
std::string Need(const char* var) {
  const char* v = std::getenv(var);
  if (v != nullptr && *v != 0) return v;
  if (laya_test::RequireGoldens()) ADD_FAILURE() << var << " is unset but LAYA_REQUIRE_GOLDENS=1";
  return "";
}

#define NEED_RIG(rig, graph, dir)                                                                   \
  const std::string graph = Need(rig.graph_env);                                                    \
  const std::string root = Need("LAYA_CHECKPOINTS");                                                \
  if (graph.empty() || root.empty()) {                                                              \
    if (!HasFailure()) GTEST_SKIP() << "no ONNX rig (local run)";                                   \
    return;                                                                                         \
  }                                                                                                 \
  const std::string dir = root + "/" + rig.checkpoint

std::mutex g_agents_mu;
std::map<std::string, std::unique_ptr<laya::agent::Agent>> g_agents;

// Closes the shared agents after the last test, so Close on a live agent is checked once per run.
// Holding them open at exit is also fine (AStaticSessionAndAgentExitCleanly).
class CloseAgents : public ::testing::Environment {
 public:
  void TearDown() override {
    std::lock_guard<std::mutex> lock(g_agents_mu);
    for (auto& kv : g_agents) EXPECT_EQ(kv.second->Close(), "");
    g_agents.clear();
  }
};
const ::testing::Environment* const g_close_agents = ::testing::AddGlobalTestEnvironment(new CloseAgents);

// One agent per checkpoint for the whole process; the tests only read it.
laya::agent::Agent* Shared(const Rig& rig, const std::string& graph, const std::string& dir) {
  auto& agents = g_agents;
  std::lock_guard<std::mutex> lock(g_agents_mu);
  auto it = agents.find(rig.checkpoint);
  if (it != agents.end()) return it->second.get();
  auto a = laya::agent::OpenOnnx(dir, graph);
  if (!a.ok()) {
    ADD_FAILURE() << "open " << rig.checkpoint << ": " << a.error();
    return nullptr;
  }
  return (agents[rig.checkpoint] = std::move(a).value()).get();
}

constexpr double kTolerance = 2e-4;       // probabilities, confidences, act_probability, noul
constexpr double kScoreTolerance = 8e-4;  // score, the expectation over the probabilities
constexpr double kDecisiveGap = 1e-3;     // the chosen label is compared when the top two differ by more
constexpr double kMaxDiffering = 0.05;    // of all compared numbers may differ at all

struct Tally {
  int numbers = 0, differing = 0, labels = 0, ties = 0, label_diffs = 0, failures = 0;
  double worst = 0;
};

void Compare(const std::string& path, const std::string& key, const Value& got, const Value& want, Tally& t) {
  if (want.kind() == Kind::Object) {
    if (got.kind() != Kind::Object || got.size() != want.size()) {
      t.failures++;
      ADD_FAILURE() << path << ": object shape differs";
      return;
    }
    for (size_t i = 0; i < want.size(); i++) {
      const Value* g = got.Find(want.key(i));
      if (g == nullptr || got.key(i) != want.key(i)) {
        t.failures++;
        ADD_FAILURE() << path << ": key " << want.key(i) << " missing or out of order";
        continue;
      }
      Compare(path + "." + want.key(i), want.key(i), *g, want.value(i), t);
    }
  } else if (want.kind() == Kind::Array) {
    if (got.kind() != Kind::Array || got.size() != want.items().size()) {
      t.failures++;
      ADD_FAILURE() << path << ": array shape differs";
      return;
    }
    for (size_t i = 0; i < want.items().size(); i++) Compare(path + "[]", key, got.items()[i], want.items()[i], t);
  } else if (want.kind() == Kind::Int || want.kind() == Kind::Float) {
    if (got.kind() != Kind::Int && got.kind() != Kind::Float) {
      t.failures++;
      ADD_FAILURE() << path << ": not a number";
      return;
    }
    const double g = laya_test::Num(got), w = laya_test::Num(want), d = std::fabs(g - w);
    t.numbers++;
    if (d > 0) t.differing++;
    t.worst = std::max(t.worst, d);
    if (d > (key == "score" ? kScoreTolerance : kTolerance)) {
      t.failures++;
      ADD_FAILURE() << path << ": got " << g << " want " << w << " (off by " << d << ")";
    }
  } else if (got.kind() != want.kind() || laya::pyvalue::Repr(got) != laya::pyvalue::Repr(want)) {
    t.failures++;
    ADD_FAILURE() << path << ": " << laya::pyvalue::Repr(got) << " want " << laya::pyvalue::Repr(want);
  }
}

Value Without(const Value& obj, const std::string& drop) {
  Value out = Value::Object();
  for (size_t i = 0; i < obj.size(); i++) {
    if (obj.key(i) != drop) out.Set(obj.key(i), obj.value(i));
  }
  return out;
}

void CompareAnswer(const std::string& path, const Value& got, const Value& want, Tally& t) {
  const Value* type = want.Find("type");
  if (type != nullptr && type->as_string() == "choice") {
    std::vector<double> ps;
    for (const Value& p : want.Find("probabilities")->items()) ps.push_back(laya_test::Num(p));
    std::sort(ps.rbegin(), ps.rend());
    if (ps.size() > 1 && ps[0] - ps[1] <= kDecisiveGap) {
      t.ties++;
      Compare(path, "", Without(got, "choice"), Without(want, "choice"), t);
      return;
    }
    t.labels++;
    const Value* g = got.Find("choice");
    if (g == nullptr || laya::pyvalue::Repr(*g) != laya::pyvalue::Repr(*want.Find("choice"))) t.label_diffs++;
  }
  Compare(path, "", got, want, t);
}

void ComparePrediction(const std::string& path, const laya::schema::Prediction& p, const Value& want, Tally& t) {
  auto json = p.ToJson();
  ASSERT_TRUE(json.ok()) << json.error();
  auto parsed = laya::pyvalue::Parse(json.value());
  ASSERT_TRUE(parsed.ok()) << parsed.error();
  const Value& got = parsed.value();
  Compare(path + ".model", "model", *got.Find("model"), *want.Find("model"), t);
  Compare(path + ".usage", "usage", *got.Find("usage"), *want.Find("usage"), t);
  const Value& ga = *got.Find("answers");
  const Value& wa = *want.Find("answers");
  ASSERT_EQ(ga.size(), wa.size()) << path;
  for (size_t i = 0; i < wa.size(); i++) {
    ASSERT_EQ(ga.key(i), wa.key(i)) << path;
    CompareAnswer(path + "." + wa.key(i), ga.value(i), wa.value(i), t);
  }
}

laya::schema::Questions QuestionsOf(const Value& v) {
  auto text = laya::pyvalue::Dumps(v);
  EXPECT_TRUE(text.ok());
  auto qs = laya::schema::Questions::Parse(text.value());
  EXPECT_TRUE(qs.ok()) << qs.error();
  return qs.ok() ? qs.value() : laya::schema::Questions::Of({}).value();
}

void Finish(const char* name, const Tally& t, int min_numbers) {
  std::printf("[ tally ] %s: %d numbers, %d differ (worst %.2e), %d decisive labels (%d differ), %d near-ties\n", name, t.numbers,
              t.differing, t.worst, t.labels, t.label_diffs, t.ties);
  EXPECT_GE(t.numbers, min_numbers) << "too few numbers were compared";
  EXPECT_LE(double(t.differing), kMaxDiffering * t.numbers) << "more of the numbers differ at all than Python's CPU noise explains";
  EXPECT_EQ(t.label_diffs, 0) << "decisive labels differ from Python";
}

class OnnxRig : public ::testing::TestWithParam<int> {};
INSTANTIATE_TEST_SUITE_P(Checkpoints, OnnxRig, ::testing::Values(0, 1), [](const auto& i) { return std::string(kRigs[i.param].checkpoint); });

TEST_P(OnnxRig, SinglePredictionsMatchPython) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  LAYA_GOLDEN_LIST(singles, rig.golden, "single");
  LAYA_EXPECT_ROWS(singles, 6);
  laya::agent::Agent* a = Shared(rig, graph, dir);
  ASSERT_NE(a, nullptr);
  Tally t;
  for (size_t i = 0; i < singles->size(); i++) {
    const Value& row = singles->value(i);
    const std::string name = singles->key(i);
    auto pred = a->Predict(*row.Find("state"), QuestionsOf(*row.Find("questions")));
    ASSERT_TRUE(pred.ok()) << name << ": " << pred.error();
    ComparePrediction(name, pred.value(), row, t);
  }
  Finish("single", t, 60);
}

TEST_P(OnnxRig, BatchPredictionsMatchPythonInInputOrder) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  LAYA_GOLDEN_LIST(batches, rig.golden, "batch");
  LAYA_EXPECT_ROWS(batches, 3);
  laya::agent::Agent* a = Shared(rig, graph, dir);
  ASSERT_NE(a, nullptr);
  Tally t;
  for (const Value& rec : batches->items()) {
    laya::agent::CallOptions opts;
    if (!rec.Find("batch_size")->is_null()) opts.batch_size = rec.Find("batch_size")->as_int();
    opts.sort_by_length = rec.Find("sort_by_length")->kind() == Kind::Bool && rec.Find("sort_by_length")->as_bool();
    const std::vector<Value>& states = rec.Find("states")->items();
    auto preds = a->PredictBatch(states, QuestionsOf(*rec.Find("questions")), opts);
    ASSERT_TRUE(preds.ok()) << preds.error();
    const std::vector<Value>& want = rec.Find("results")->items();
    ASSERT_EQ(preds.value().size(), want.size());
    ASSERT_EQ(want.size(), 9u);
    for (size_t i = 0; i < want.size(); i++) ComparePrediction("batch/state" + std::to_string(i), preds.value()[i], want[i], t);
  }
  Finish("batch", t, 240);
}

TEST_P(OnnxRig, BatchOrderFollowsTheInputWhateverTheSortAndBatchSize) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  LAYA_GOLDEN_LIST(batches, rig.golden, "batch");
  laya::agent::Agent* a = Shared(rig, graph, dir);
  ASSERT_NE(a, nullptr);
  const Value& rec = batches->items()[0];
  std::vector<Value> states = rec.Find("states")->items();
  const auto qs = QuestionsOf(*rec.Find("questions"));
  auto fwd = a->PredictBatch(states, qs);
  ASSERT_TRUE(fwd.ok()) << fwd.error();
  std::vector<Value> rev(states.rbegin(), states.rend());
  laya::agent::CallOptions o;
  o.batch_size = 2;
  o.sort_by_length = true;
  auto back = a->PredictBatch(rev, qs, o);
  ASSERT_TRUE(back.ok()) << back.error();
  ASSERT_EQ(fwd.value().size(), back.value().size());
  const size_t n = states.size();
  for (size_t i = 0; i < n; i++) {
    auto x = fwd.value()[i].ToJson(), y = back.value()[n - 1 - i].ToJson();
    ASSERT_TRUE(x.ok() && y.ok());
    auto px = laya::pyvalue::Parse(x.value()), py = laya::pyvalue::Parse(y.value());
    Tally t;
    Compare("state" + std::to_string(i), "", py.value(), px.value(), t);  // within the numeric tolerances
    EXPECT_EQ(t.failures, 0) << "state " << i << " changed with its position in the batch";
  }
}

TEST_P(OnnxRig, ConcurrentPredictAgreesWithASerialCall) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  LAYA_GOLDEN_LIST(singles, rig.golden, "single");
  laya::agent::Agent* a = Shared(rig, graph, dir);
  ASSERT_NE(a, nullptr);
  const size_t n = singles->size();
  std::vector<laya::schema::Questions> qs;
  std::vector<std::string> serial;
  for (size_t i = 0; i < n; i++) {
    qs.push_back(QuestionsOf(*singles->value(i).Find("questions")));
    auto p = a->Predict(*singles->value(i).Find("state"), qs.back());
    ASSERT_TRUE(p.ok()) << p.error();
    serial.push_back(p.value().ToJson().value());
  }
  std::atomic<int> bad{0};
  std::vector<std::thread> threads;
  for (int th = 0; th < 8; th++) {
    threads.emplace_back([&, th] {
      for (int round = 0; round < 3; round++) {
        for (size_t k = 0; k < n; k++) {
          const size_t i = (k + size_t(th)) % n;
          auto p = a->Predict(*singles->value(i).Find("state"), qs[i]);
          if (!p.ok() || p.value().ToJson().value() != serial[i]) bad++;
        }
      }
    });
  }
  for (auto& t : threads) t.join();
  EXPECT_EQ(bad.load(), 0);
}

TEST_P(OnnxRig, ACancelledCallStopsBeforeTheRuntime) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  LAYA_GOLDEN_LIST(singles, rig.golden, "single");
  laya::agent::Agent* a = Shared(rig, graph, dir);
  ASSERT_NE(a, nullptr);
  laya::agent::CallOptions o;
  o.cancel.Cancel();
  auto p = a->Predict(*singles->value(0).Find("state"), QuestionsOf(*singles->value(0).Find("questions")), o);
  ASSERT_FALSE(p.ok());
  EXPECT_TRUE(laya::agent::IsCancelled(p.error())) << p.error();
}

TEST_P(OnnxRig, SessionRunIsGuardedAgainstHostileBatches) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  laya::infer::Session& session = *s.value();
  const laya::CancelToken live;

  auto ok = session.Run(Tiny(2, 3), live);
  ASSERT_TRUE(ok.ok()) << ok.error();
  EXPECT_EQ(ok.value().logits.size(), 6u);
  EXPECT_EQ(ok.value().act_logits.size(), 4u);

  Batch wrong_rank = Tiny(2, 3);
  wrong_rank.input_ids.pop_back();  // a flattened slice that no longer matches rows*length
  EXPECT_FALSE(session.Run(wrong_rank, live).ok());

  Batch huge = Tiny(1, 3);
  huge.rows = int64_t(1) << 40;  // far past the cap: refused before any allocation
  EXPECT_FALSE(session.Run(huge, live).ok());
  huge = Tiny(1, 3);
  huge.length = int64_t(1) << 40;
  EXPECT_FALSE(session.Run(huge, live).ok());
  Batch neg = Tiny(1, 3);
  neg.length = -1;
  EXPECT_FALSE(session.Run(neg, live).ok());

  Batch one_marker = Tiny(1, 1);  // the exported head takes topk(2)
  EXPECT_FALSE(session.Run(one_marker, live).ok());
  Batch oob = Tiny(1, 3);
  oob.marker_pos[0] = 99;
  EXPECT_FALSE(session.Run(oob, live).ok());
  Batch mask2 = Tiny(1, 3);
  mask2.marker_mask[1] = 2;  // not a bool
  EXPECT_FALSE(session.Run(mask2, live).ok());
  Batch bad_q = Tiny(1, 3);
  bad_q.qtype[0] = 7;
  EXPECT_FALSE(session.Run(bad_q, live).ok());
  Batch empty;
  EXPECT_FALSE(session.Run(empty, live).ok());

  // Passes every check of ours and fails inside ONNX Runtime (a token id past the vocabulary): the
  // failure must come back as a Result, not an exception.
  Batch past_vocab = Tiny(1, 3);
  past_vocab.input_ids[0] = 1000000000;
  const auto pv = session.Run(past_vocab, live);
  ASSERT_FALSE(pv.ok());
  EXPECT_EQ(pv.error().rfind("onnx: run:", 0), 0u) << pv.error();

  laya::CancelToken fired;
  fired.Cancel();
  auto c = session.Run(Tiny(1, 3), fired);
  ASSERT_FALSE(c.ok());
  EXPECT_TRUE(laya::infer::IsCancelled(c.error()));

  // The session still works after every refusal.
  EXPECT_TRUE(session.Run(Tiny(1, 3), live).ok());
}

TEST_P(OnnxRig, CloseIsIdempotentAndRunAfterCloseIsAnError) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  EXPECT_EQ(s.value()->Close(), "");
  EXPECT_EQ(s.value()->Close(), "");
  auto r = s.value()->Run(Tiny(1, 3), laya::CancelToken());
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("closed"), std::string::npos) << r.error();
}

// A batch that keeps the runtime busy for seconds.
Batch Long(int64_t rows, int64_t length) {
  Batch b = Tiny(rows, 3);
  b.length = length;
  b.input_ids.assign(size_t(rows * length), 5);
  b.attention_mask.assign(size_t(rows * length), 1);
  return b;
}
using Clock = std::chrono::steady_clock;
double Since(Clock::time_point t) { return std::chrono::duration<double>(Clock::now() - t).count(); }
constexpr int64_t kLongRows = 16, kLongLength = 192;

// Seconds one uncancelled long run takes; the tests below only mean something if it is long.
double LongRunSeconds(laya::infer::Session& s) {
  const auto t = Clock::now();
  const auto r = s.Run(Long(kLongRows, kLongLength), laya::CancelToken());
  EXPECT_TRUE(r.ok()) << (r.ok() ? "" : r.error());
  return Since(t);
}

TEST_P(OnnxRig, ACancelDuringARunStopsTheRun) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  const double full = LongRunSeconds(*s.value());
  ASSERT_GT(full, 0.8) << "the long batch is too short to tell a mid-run cancel from a late one";
  laya::CancelToken token;
  std::thread canceller([&] {
    std::this_thread::sleep_for(std::chrono::milliseconds(50));
    token.Cancel();
  });
  const auto t = Clock::now();
  const auto r = s.value()->Run(Long(kLongRows, kLongLength), token);
  const double took = Since(t);
  canceller.join();
  ASSERT_FALSE(r.ok());
  EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();
  EXPECT_LT(took, full / 3) << "cancelled run took " << took << " s of " << full << " s";
  EXPECT_TRUE(s.value()->Run(Tiny(1, 3), laya::CancelToken()).ok());  // the session is still usable
}

TEST_P(OnnxRig, ACancelThatLandsAfterTheRunStillReportsCancelled) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  // The token fires inside the post-run window: the run is tiny, so a watcher cannot beat it
  // reliably; either outcome is allowed only if it is cancelled or a complete correct result.
  for (int i = 0; i < 50; i++) {
    laya::CancelToken token;
    std::thread canceller([&] { token.Cancel(); });
    const auto r = s.value()->Run(Tiny(2, 3), token);
    canceller.join();
    if (!r.ok()) EXPECT_TRUE(laya::infer::IsCancelled(r.error())) << r.error();
    else EXPECT_EQ(r.value().logits.size(), 6u);
  }
}

TEST_P(OnnxRig, CloseInterruptsARunInFlight) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  const double full = LongRunSeconds(*s.value());
  ASSERT_GT(full, 0.8);
  std::atomic<bool> failed{false};
  std::thread runner([&] { failed = !s.value()->Run(Long(kLongRows, kLongLength), laya::CancelToken()).ok(); });
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
  const auto t = Clock::now();
  EXPECT_EQ(s.value()->Close(), "");
  const double took = Since(t);
  runner.join();
  EXPECT_TRUE(failed.load());
  EXPECT_LT(took, full / 3) << "Close took " << took << " s of a " << full << " s run";
}

TEST_P(OnnxRig, CloseReturnsPromptlyUnderSustainedRunTraffic) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto s = laya::onnx::Open(graph);
  ASSERT_TRUE(s.ok()) << s.error();
  const auto t0 = Clock::now();
  ASSERT_TRUE(s.value()->Run(Long(4, 128), laya::CancelToken()).ok());
  const double one = Since(t0);  // one such run alone
  ASSERT_GT(one, 0.15);
  std::atomic<bool> stop{false};
  std::atomic<int> ran{0}, refused{0}, other{0};
  std::vector<std::thread> threads;
  for (int i = 0; i < 4; i++) {
    threads.emplace_back([&] {
      while (!stop.load()) {
        const auto r = s.value()->Run(Long(4, 128), laya::CancelToken());
        if (r.ok()) ran++;
        else if (r.error().find("closed") != std::string::npos || r.error().find("run:") != std::string::npos) refused++;
        else other++;
      }
    });
  }
  std::this_thread::sleep_for(std::chrono::milliseconds(300));
  const auto t = Clock::now();
  EXPECT_EQ(s.value()->Close(), "");
  const double took = Since(t);
  stop = true;
  for (auto& th : threads) th.join();
  EXPECT_GT(ran.load() + refused.load(), 0);  // every thread ended on a Result, none on a hang
  EXPECT_EQ(other.load(), 0);
  EXPECT_LT(took, one * 2) << "Close took " << took << " s with one run alone taking " << one << " s";
}

// A user program may hold a Session and an Agent in statics; it must exit cleanly, closed or not.
TEST_P(OnnxRig, AStaticSessionAndAgentExitCleanly) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  // The child gets no variable that turns the runtime's telemetry off, so it exits with the uploader
  // thread running unless Open opts out (the abort this test guards against).
  const std::set<std::string> drop = {"ORT_DISABLE_TELEMETRY", "ORT_RUNNING_UNIT_TESTS", "CI", "TF_BUILD",
                                      "GITHUB_ACTIONS", "GITLAB_CI", "CIRCLECI", "TRAVIS", "JENKINS_URL",
                                      "CODEBUILD_BUILD_ID", "BUILDKITE", "TEAMCITY_VERSION", "APPVEYOR",
                                      "BITBUCKET_BUILD_NUMBER", "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"};
  std::vector<const char*> envp;
  for (char** e = environ; *e != nullptr; e++) {
    const char* eq = std::strchr(*e, '=');
    if (eq == nullptr || drop.count(std::string(*e, eq - *e)) == 0) envp.push_back(*e);
  }
  envp.push_back(nullptr);
  for (const char* mode : {"", "close"}) {
    const pid_t pid = fork();
    ASSERT_GE(pid, 0);
    if (pid == 0) {
      const char* argv[] = {LAYA_STATIC_HOLDER, graph.c_str(), dir.c_str(), mode, nullptr};
      if (*mode == 0) argv[3] = nullptr;
      execve(LAYA_STATIC_HOLDER, const_cast<char* const*>(argv), const_cast<char* const*>(envp.data()));
      _exit(127);
    }
    int status = 0;
    ASSERT_EQ(waitpid(pid, &status, 0), pid);
    EXPECT_TRUE(WIFEXITED(status)) << "mode '" << mode << "' died by signal " << (WIFSIGNALED(status) ? WTERMSIG(status) : 0);
    EXPECT_EQ(WEXITSTATUS(status), 0) << "mode '" << mode << "'";
  }
}

TEST_P(OnnxRig, SessionsOpenAndCloseWhileOthersRun) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  auto first = laya::onnx::Open(graph);
  ASSERT_TRUE(first.ok()) << first.error();
  std::atomic<int> bad{0};
  std::thread worker([&] {
    for (int i = 0; i < 20; i++) {
      if (!first.value()->Run(Tiny(2, 3), laya::CancelToken()).ok()) bad++;
    }
  });
  for (int i = 0; i < 3; i++) {
    laya::onnx::Options o;
    o.intra_op_threads = 1;
    auto other = laya::onnx::Open(graph, o);  // a second session, dropped without Close
    if (!other.ok() || !other.value()->Run(Tiny(1, 3), laya::CancelToken()).ok()) bad++;
  }
  worker.join();
  EXPECT_EQ(bad.load(), 0);
}

TEST_P(OnnxRig, OpenOnnxWithAWrongCheckpointDirIsAnError) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  TempDir empty;
  const auto a = laya::agent::OpenOnnx(empty.str(), graph);
  EXPECT_FALSE(a.ok());
}

TEST_P(OnnxRig, TheGraphWithoutItsDataFileIsAnError) {
  const Rig& rig = kRigs[GetParam()];
  NEED_RIG(rig, graph, dir);
  TempDir d;
  fs::copy_file(graph, d.str() + "/laya.onnx");  // the .data file stays behind
  const auto r = laya::onnx::Open(d.str() + "/laya.onnx");
  ASSERT_FALSE(r.ok());
  EXPECT_NE(r.error().find("cannot open"), std::string::npos) << r.error();
}

}  // namespace