Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

15.8 KB
Raw Blame
#include "laya/agent.hpp"

#include <algorithm>
#include <exception>
#include <fstream>
#include <sstream>

#include "laya/collate.hpp"
#include "laya/unicodedata.hpp"

namespace laya {
namespace agent {

const char* const kClosed = "agent: agent is closed";
const char* const kCancelled = "agent: call cancelled";

bool IsClosed(const std::string& e) { return e == kClosed; }
bool IsCancelled(const std::string& e) { return e == kCancelled; }

namespace {

using pyvalue::Value;
using Kind = Value::Kind;
constexpr int64_t kLookaheadBatches = 8;
const char* const kConfigFile = "rl_agent_config.json";

std::string Num(double d) {
  std::string s = pyvalue::Repr(Value::Float(d));
  if (s.size() > 2 && s.compare(s.size() - 2, 2, ".0") == 0) s.resize(s.size() - 2);
  return s;
}

// Whether the raw value is exactly what clamp_temperature returned.
bool SameNumber(const Value& raw, double applied) {
  switch (raw.kind()) {
    case Kind::String: {
      auto f = decode::PyFloat(raw.as_string());
      return f && *f == applied;
    }
    case Kind::Int: return double(raw.as_int()) == applied;
    case Kind::Float: return raw.as_float() == applied;
    default: return false;  // a bool is no number to clamp_temperature
  }
}

Result<int64_t> LengthField(const Value& obj, const char* key, int64_t dflt) {
  using R = Result<int64_t>;
  const Value* v = obj.Find(key);
  if (v == nullptr) return R::Ok(dflt);
  const std::string name = std::string("agent: ") + kConfigFile + ": " + key + ": ";
  if (v->kind() == Kind::BigInt && v->big_int().rfind('-', 0) != 0) return R::Fail(name + "too large for an int");
  if (v->kind() != Kind::Int || v->as_int() <= 0) return R::Fail(name + "want a positive integer, got " + pyvalue::Repr(*v));
  if (v->as_int() > kMaxSequenceLength) return R::Fail(name + "too large (the limit is " + std::to_string(kMaxSequenceLength) + ")");
  return R::Ok(v->as_int());
}


}  // namespace

namespace {

// Closes the session on every failure path of Open; empty once the agent owns it.
struct SessionGuard {
  std::unique_ptr<infer::Session>& s;
  ~SessionGuard() {
    if (!s) return;
    try {
      s->Close();
    } catch (...) {
    }
  }
};

}  // namespace

Result<std::unique_ptr<Agent>> Agent::Open(const std::string& dir, std::unique_ptr<infer::Session> session) {
  using R = Result<std::unique_ptr<Agent>>;
  if (!session) return R::Fail("agent: Open needs a session");
  SessionGuard guard{session};
  std::ifstream in(dir + "/" + kConfigFile, std::ios::binary);
  if (!in) return R::Fail(std::string("agent: ") + kConfigFile + ": cannot read " + dir + "/" + kConfigFile);
  std::stringstream ss;
  ss << in.rdbuf();
  auto tok = tokenizer::Tokenizer::LoadDir(dir);
  if (!tok.ok()) return R::Fail("agent: tokenizer: " + tok.error());
  return OpenParts(ss.str(), std::move(tok).value(), std::move(session));
}

Result<std::unique_ptr<Agent>> Agent::OpenParts(std::string_view config_json, tokenizer::Tokenizer tok,
                                                std::unique_ptr<infer::Session> session) {
  using R = Result<std::unique_ptr<Agent>>;
  if (!session) return R::Fail("agent: Open needs a session");
  SessionGuard guard{session};
  const std::string where = std::string("agent: ") + kConfigFile + ": ";
  auto parsed = pyvalue::Parse(config_json);
  if (!parsed.ok()) return R::Fail(where + parsed.error());
  const Value& obj = parsed.value();
  if (obj.kind() != Kind::Object) return R::Fail(where + "want a JSON object, got " + pyvalue::TypeName(obj));
  std::unique_ptr<Agent> a(new Agent());
  if (const Value* l = obj.Find("option_layout")) {
    if (l->kind() != Kind::String || l->as_string() != "sequential") {
      return R::Fail(where + "option_layout: option_layout is not supported by the ONNX runtime: " + pyvalue::Repr(*l));
    }
  }
  auto max_len = LengthField(obj, "max_len", 512);
  if (!max_len.ok()) return R::Fail(max_len.error());
  auto head_max_len = LengthField(obj, "head_max_len", 192);
  if (!head_max_len.ok()) return R::Fail(head_max_len.error());
  a->cfg_.max_len = max_len.value();
  a->cfg_.head_max_len = head_max_len.value();

  const Value* by_type = obj.Find("temperature");
  if (by_type != nullptr && by_type->kind() != Kind::Array) {
    return R::Fail(where + "temperature: want a list of 3 numbers, got " + pyvalue::Repr(*by_type));
  }
  const Value* by_options = obj.Find("temperature_by_options");
  if (by_options != nullptr && by_options->kind() != Kind::Object) {
    return R::Fail(where + "temperature_by_options: want an object of bucket to number, got " + pyvalue::Repr(*by_options));
  }
  auto temps = decode::Temperatures::Of(by_type, by_options);
  if (!temps.ok()) return R::Fail(where + "temperature: " + temps.error());
  a->temps_ = std::move(temps).value();
  std::vector<std::string> rejected;
  if (by_options != nullptr) {
    for (size_t i = 0; i < by_options->size(); i++) {
      const double applied = a->temps_.by_options().at(by_options->key(i));
      if (!SameNumber(by_options->value(i), applied)) {
        rejected.push_back(by_options->key(i) + "=" + pyvalue::Repr(by_options->value(i)) + " -> " + Num(applied));
      }
    }
  }
  if (by_type != nullptr) {
    for (size_t i = 0; i < by_type->size(); i++) {
      if (!SameNumber(by_type->items()[i], a->temps_.by_type()[i])) {
        rejected.push_back("temperature[" + std::to_string(i) + "]=" + pyvalue::Repr(by_type->items()[i]) + " -> " + Num(a->temps_.by_type()[i]));
      }
    }
  }
  if (!rejected.empty()) {
    std::string joined;
    for (size_t i = 0; i < rejected.size(); i++) joined += (i ? ", " : "") + rejected[i];
    a->cfg_.warnings.push_back("this checkpoint ships temperatures outside [0.5, 5] which would distort confidence; clamping " +
                               joined + ". Treat confidence from the affected buckets as uncalibrated.");
  }

  const std::optional<int32_t> pad = tok.Pad(), cls = tok.Cls(), sep = tok.Sep(), mask = tok.Mask();
  if (!pad) return R::Fail("agent: missing special token: no pad token");
  if (!cls) return R::Fail("agent: missing special token: no CLS token");
  if (!sep) return R::Fail("agent: missing special token: no SEP token");
  if (!mask) return R::Fail("agent: missing special token: no mask token");
  a->pad_id_ = *pad;
  a->tok_ = std::move(tok);
  a->sess_ = std::move(session);
  return R::Ok(std::move(a));
}

Agent::~Agent() { Close(); }

bool Agent::Enter() const {
  std::lock_guard<std::mutex> lock(mu_);
  if (shut_ || !sess_) return false;
  inflight_++;
  return true;
}

void Agent::Leave() const {
  std::lock_guard<std::mutex> lock(mu_);
  if (--inflight_ == 0) cv_.notify_all();
}

std::string Agent::Close() {
  std::unique_lock<std::mutex> lock(mu_);
  shut_ = true;
  closed_.store(true);
  cv_.wait(lock, [&] { return inflight_ == 0; });
  if (close_started_) {
    cv_.wait(lock, [&] { return close_done_; });
    return close_err_;
  }
  close_started_ = true;
  lock.unlock();
  std::string err;
  try {
    if (sess_) err = sess_->Close();
  } catch (const std::exception& e) {
    err = std::string("agent: session Close threw: ") + e.what();
  } catch (...) {
    err = "agent: session Close threw";
  }
  lock.lock();
  close_err_ = err;
  close_done_ = true;
  cv_.notify_all();
  return close_err_;
}

Result<schema::Prediction> Agent::Predict(const Value& state, const schema::Questions& qs, const CallOptions& opts) const {
  using R = Result<schema::Prediction>;
  auto r = PredictBatch(std::vector<Value>{state}, qs, opts);
  if (!r.ok()) {
    // The top-level error only: drop the "agent: state 0: " prefix a one-state call has no use for.
    std::string e = r.error();
    const std::string prefix = "agent: state 0: ";
    if (e.compare(0, prefix.size(), prefix) == 0) e = "agent: " + e.substr(prefix.size());
    return R::Fail(e);
  }
  return R::Ok(std::move(r).value()[0]);
}

Result<std::vector<schema::Prediction>> Agent::PredictBatch(const std::vector<Value>& states, const schema::Questions& qs,
                                                            const CallOptions& opts) const {
  using R = Result<std::vector<schema::Prediction>>;
  if (opts.cancel.cancelled()) return R::Fail(kCancelled);
  if (int64_t(states.size()) > kMaxStates) {
    return R::Fail("agent: " + std::to_string(states.size()) + " states in one call, the limit is " + std::to_string(kMaxStates));
  }
  if (opts.max_len > kMaxSequenceLength || opts.head_max_len > kMaxSequenceLength) {
    return R::Fail("agent: max_len and head_max_len are limited to " + std::to_string(kMaxSequenceLength));
  }
  if (!Enter()) return R::Fail(kClosed);
  struct Leaver {
    const Agent* a;
    ~Leaver() { a->Leave(); }
  } leaver{this};
  for (size_t i = 0; i < states.size(); i++) {
    if (states[i].is_null()) return R::Fail("agent: state " + std::to_string(i) + ": state is null; pass a string, an object or a list");
  }
  if (states.empty()) return R::Ok({});
  if (qs.size() == 0) {
    std::vector<schema::Prediction> preds(states.size());
    for (auto& p : preds) p.usage.bare = true;  // what the Python ONNX agent returns with no questions
    return R::Ok(std::move(preds));
  }
  return Chunks(states, qs, opts);
}

Result<std::vector<sequence::Sequence>> Agent::Encode(const Value& state, const schema::Questions& qs, const CallOptions& c) const {
  using R = Result<std::vector<sequence::Sequence>>;
  auto st = sequence::EncodeState(tok_, state);
  if (!st.ok()) return R::Fail(st.error());
  sequence::Budget budget{cfg_.max_len, cfg_.head_max_len};
  if (c.max_len > 0) budget.max_len = c.max_len;
  if (c.head_max_len > 0) budget.head_max_len = c.head_max_len;
  std::vector<sequence::Sequence> seqs;
  seqs.reserve(qs.size());
  for (size_t i = 0; i < qs.size(); i++) {
    sequence::MarkersError me;
    auto s = sequence::Build(tok_, qs.at(i).question, st.value(), budget, &me);
    if (!s.ok()) return R::Fail(me.max_len != 0 ? me.Message(qs.at(i).id) : s.error());
    seqs.push_back(std::move(s).value());
  }
  return R::Ok(std::move(seqs));
}

Result<std::vector<schema::Prediction>> Agent::Chunks(const std::vector<Value>& states, const schema::Questions& qs,
                                                      const CallOptions& c) const {
  using R = Result<std::vector<schema::Prediction>>;
  const size_t n = states.size();
  size_t chunk = n;
  if (c.batch_size > 0) chunk = std::min(size_t(c.batch_size), n);
  const bool reorder = c.sort_by_length && 1 < chunk && chunk < n;
  const size_t window = reorder ? chunk * size_t(kLookaheadBatches) : chunk;
  std::vector<schema::Prediction> results(n);
  for (size_t start = 0; start < n; start += window) {
    const size_t len = std::min(window, n - start);
    Encoded encoded(len);
    for (size_t i = 0; i < len; i++) {
      if (c.cancel.cancelled()) return R::Fail(kCancelled);
      auto e = Encode(states[start + i], qs, c);
      if (!e.ok()) return R::Fail("agent: state " + std::to_string(start + i) + ": " + e.error());
      encoded[i] = std::move(e).value();
    }
    std::vector<size_t> order(len);
    for (size_t i = 0; i < len; i++) order[i] = i;
    if (reorder) {
      std::vector<size_t> longest(len, 0);
      for (size_t i = 0; i < len; i++) {
        for (const auto& s : encoded[i]) longest[i] = std::max(longest[i], s.ids.size());
      }
      std::stable_sort(order.begin(), order.end(), [&](size_t x, size_t y) { return longest[x] < longest[y]; });
    }
    std::vector<size_t> sizes;  // states per run
    if (c.batch_size > 0) {
      for (size_t off = 0; off < len; off += chunk) sizes.push_back(std::min(chunk, len - off));
    } else {
      std::vector<int64_t> length(len), width(len);
      for (size_t i = 0; i < len; i++) {
        for (const auto& s : encoded[order[i]]) {
          length[i] = std::max(length[i], int64_t(s.ids.size()));
          width[i] = std::max(width[i], int64_t(s.markers.size()));
        }
      }
      sizes = collate::PlanRuns(qs.size(), length, width);
    }
    size_t off = 0;
    for (size_t take : sizes) {
      if (c.cancel.cancelled()) return R::Fail(kCancelled);
      if (closed_.load()) return R::Fail(kClosed);
      std::vector<size_t> idx(order.begin() + std::ptrdiff_t(off), order.begin() + std::ptrdiff_t(off + take));
      off += take;
      auto preds = Run(qs, encoded, idx, c);
      if (!preds.ok()) return R::Fail(preds.error());
      for (size_t j = 0; j < idx.size(); j++) results[start + idx[j]] = std::move(preds.value()[j]);
    }
  }
  return R::Ok(std::move(results));
}

namespace {

schema::Usage UsageOf(const schema::Questions& qs, const std::vector<sequence::Sequence>& seqs, int64_t input_tokens) {
  schema::Usage u;
  u.input_tokens = input_tokens;
  u.state_tokens = seqs[0].truncation.state_tokens;
  for (size_t i = 0; i < seqs.size(); i++) {
    const std::string& id = qs.at(i).id;
    const sequence::Sequence& s = seqs[i];
    u.state_tokens_dropped = std::max(u.state_tokens_dropped, s.truncation.state_tokens_dropped);
    if (s.truncation.truncated) u.truncated_questions.push_back(id);
    if (s.stats.Collapsed()) {
      schema::CollapsedQuestion c;
      c.id = id;
      c.options.total = s.stats.options;
      c.options.distinct = s.stats.options_distinct;
      c.options.tokens_per_option = s.stats.tokens_per_option;
      u.collapsed_options.push_back(std::move(c));
    }
  }
  u.truncated = u.state_tokens_dropped > 0;
  return u;
}

}  // namespace

Result<std::vector<schema::Prediction>> Agent::Run(const schema::Questions& qs, const Encoded& encoded,
                                                   const std::vector<size_t>& idx, const CallOptions& c) const {
  using R = Result<std::vector<schema::Prediction>>;
  std::vector<collate::Row> rows;
  rows.reserve(idx.size() * qs.size());
  for (size_t i : idx) {
    for (size_t r = 0; r < encoded[i].size(); r++) {
      rows.push_back({encoded[i][r].ids, encoded[i][r].markers, qs.at(r).question.type()});
    }
  }
  auto batch = collate::Collate(rows, pad_id_);
  if (!batch.ok()) return R::Fail("agent: " + batch.error());
  const infer::Batch& b = batch.value();
  auto fail = [&](const std::string& why) {
    return R::Fail("agent: running " + std::to_string(b.rows) + " rows of " + std::to_string(b.length) + " tokens with " +
                   std::to_string(b.markers) + " marker slots: " + why);
  };
  if (std::string bad = b.Validate(); !bad.empty()) return fail(bad);
  Result<infer::Output> out = Result<infer::Output>::Fail("");
  try {
    out = sess_->Run(b, c.cancel);
  } catch (const std::exception& e) {
    return fail(std::string("session threw: ") + e.what());
  } catch (...) {
    return fail("session threw");
  }
  if (!out.ok()) return infer::IsCancelled(out.error()) ? R::Fail(kCancelled) : fail(out.error());
  if (std::string bad = out.value().Check(b); !bad.empty()) return fail(bad);
  const infer::Output& o = out.value();

  std::vector<schema::Prediction> preds(idx.size());
  size_t row = 0;
  for (size_t j = 0; j < idx.size(); j++) {
    const auto& seqs = encoded[idx[j]];
    int64_t input_tokens = 0;
    for (size_t r = 0; r < seqs.size(); r++) {
      const size_t at = row + r;
      for (int64_t m = 0; m < b.length; m++) input_tokens += b.attention_mask[at * size_t(b.length) + size_t(m)];
      const auto& e = qs.at(r);
      auto act = decode::ActionProbabilities({o.act_logits[at * 2], o.act_logits[at * 2 + 1]});
      if (!act.ok()) return fail("action logits of question " + e.id + ": " + act.error());
      decode::Row dr;
      dr.logits.assign(o.logits.begin() + std::ptrdiff_t(at * size_t(b.markers)),
                       o.logits.begin() + std::ptrdiff_t((at + 1) * size_t(b.markers)));
      dr.k = int64_t(seqs[r].markers.size());
      dr.act = act.value()[0];
      auto ans = decode::Decode(e.question, dr, "", &temps_);
      if (!ans.ok()) return fail("question " + e.id + ": " + ans.error());
      preds[j].answers.push_back({e.id, std::move(ans).value()});
    }
    preds[j].usage = UsageOf(qs, seqs, input_tokens);
    row += seqs.size();
  }
  return R::Ok(std::move(preds));
}

}  // namespace agent
}  // namespace laya