Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

8.2 KB
Raw Blame
// Locked-layer fetch for hybrid decode (python/freetoken/moe/locked_fetch.py).
//
// A locked layer's host banks have no device address, so the GPU cannot gather its rows. Per decode step a publish kernel
// writes the fetch list (count + layer-local expert rows) into mapped meta and rings a doorbell (GpuSync's words); the
// coordinator and worker threads memcpy each listed row of every bank into pinned staging, then complete the doorbell,
// and fast_index_copy_multi copies staging row i into the i-th evicted slot after a stream memop wait, or with its own
// spin wait where memops do not work. Everything is kernels and memops, so it captures into the decode graph. Row i of a
// bank sits at i * the layer's own row bytes, so layers of different widths share staging.

#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstring>
#include <memory>
#include <stdexcept>
#include <thread>
#include <vector>

#include <torch/extension.h>

#include <freetoken/hip_compat.h>

#include "gpu_sync.h"
#include "locked_fetch_io.h"
#include "threads.h"

namespace freetoken::cpu_moe {
namespace {

using fetch_clock = std::chrono::steady_clock;
// same idle policy as the MoE coordinator: spin while decode traffic flows, sleep once it stops
constexpr auto kHotWindow = std::chrono::milliseconds(50);
constexpr int64_t kChunkBytes = 256 << 10;

struct CopyJob {
  char* dst;
  const char* src;
  int64_t bytes;
};

class LockedFetch {
 public:
  // src[l][b] / row_bytes[l][b]: host base and row bytes of layer l's bank b (0 for banks or layers it never fetches).
  // memops: wait on the doorbell with a stream memop, else the gather spins on done_word().
  LockedFetch(std::vector<std::vector<uintptr_t>> src, std::vector<std::vector<int64_t>> row_bytes, std::vector<uintptr_t> staging,
              int max_rows, std::vector<int> cpus, bool memops)
      : src_(std::move(src)), row_bytes_(std::move(row_bytes)), staging_(std::move(staging)), max_rows_(max_rows), memops_(memops) {
    sync_ = std::make_unique<GpuSync>(memops);
    void* meta = nullptr;
    void* meta_dev = nullptr;
    const size_t meta_bytes = sizeof(int64_t) * (1 + (max_rows + 1) / 2);
    if (cudaHostAlloc(&meta, meta_bytes, cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess ||
        cudaHostGetDevicePointer(&meta_dev, meta, 0) != cudaSuccess) {
      (void)cudaGetLastError();
      if (meta) (void)cudaFreeHost(meta);
      throw std::runtime_error("locked fetch: mapped meta allocation failed");
    }
    meta_ = meta;
    count_ = static_cast<volatile int64_t*>(meta);
    rows_ = reinterpret_cast<const int32_t*>(static_cast<int64_t*>(meta) + 1);
    count_dev_ = static_cast<int64_t*>(meta_dev);
    rows_dev_ = reinterpret_cast<int32_t*>(static_cast<int64_t*>(meta_dev) + 1);
    const int coord_cpu = cpus.empty() ? -1 : cpus[0];
    for (size_t i = 1; i < cpus.size(); ++i) workers_.emplace_back([this, cpu = cpus[i]] { worker_loop(cpu); });
    coord_ = std::thread([this, coord_cpu] { coordinator_loop(coord_cpu); });
  }

  ~LockedFetch() {
    stop_.store(true);
    coord_.join();
    for (auto& t : workers_) t.join();
    (void)cudaFreeHost(meta_);
  }

  LockedFetch(const LockedFetch&) = delete;
  LockedFetch& operator=(const LockedFetch&) = delete;

  // Hands one layer step's fetch list (the hybrid ensure's num_indices and src_indices) to the copy threads, then waits for
  // them on `stream` when memops work; otherwise the gather that follows waits on done_word() itself.
  void publish(uintptr_t stream, int layer, uintptr_t num_indices, uintptr_t src_indices) {
    locked_publish(stream, reinterpret_cast<const int64_t*>(num_indices), reinterpret_cast<const int32_t*>(src_indices), max_rows_,
                   count_dev_, rows_dev_, reinterpret_cast<volatile int64_t*>(sync_->ready_word()),
                   reinterpret_cast<volatile int64_t*>(sync_->done_word()), static_cast<int64_t>(layer) + 1);
    if (memops_) sync_->gpu_wait(stream);
  }
  uintptr_t done_word() const { return sync_->done_word(); }

 private:
  void run_jobs() {
    for (;;) {
      const int64_t i = next_.fetch_add(1, std::memory_order_relaxed);
      if (i >= static_cast<int64_t>(jobs_.size())) return;
      std::memcpy(jobs_[i].dst, jobs_[i].src, static_cast<size_t>(jobs_[i].bytes));
    }
  }

  void worker_loop(int cpu) {
    if (cpu >= 0) pin_this_thread(cpu);
    uint64_t seen = 0;
    auto last_active = fetch_clock::now();
    int64_t sleep_us = 20;
    while (!stop_.load(std::memory_order_relaxed)) {
      const uint64_t gen = gen_.load(std::memory_order_acquire);
      if (gen == seen) {
        if (fetch_clock::now() - last_active < kHotWindow) {
          cpu_relax();
        } else {
          std::this_thread::sleep_for(std::chrono::microseconds(sleep_us));
          sleep_us = std::min<int64_t>(sleep_us * 2, 2000);
        }
        continue;
      }
      seen = gen;
      run_jobs();
      finished_.fetch_add(1, std::memory_order_acq_rel);
      last_active = fetch_clock::now();
      sleep_us = 20;
    }
  }

  // Splits this step's rows into chunk jobs and runs them with the workers.
  void copy_layer(int layer) {
    int64_t n = *count_;
    n = std::max<int64_t>(0, std::min<int64_t>(n, max_rows_));
    jobs_.clear();
    const auto& src = src_[layer];
    const auto& row_bytes = row_bytes_[layer];
    for (int64_t i = 0; i < n; ++i) {
      const int64_t row = rows_[i];
      for (size_t b = 0; b < staging_.size(); ++b) {
        const int64_t rb = row_bytes[b];
        if (rb == 0) continue;
        char* dst = reinterpret_cast<char*>(staging_[b]) + i * rb;
        const char* s = reinterpret_cast<const char*>(src[b]) + row * rb;
        for (int64_t off = 0; off < rb; off += kChunkBytes) jobs_.push_back({dst + off, s + off, std::min(kChunkBytes, rb - off)});
      }
    }
    if (jobs_.empty()) return;
    next_.store(0, std::memory_order_relaxed);
    finished_.store(0, std::memory_order_relaxed);
    gen_.fetch_add(1, std::memory_order_acq_rel);
    run_jobs();
    // on stop a worker may leave before it joins this generation; the coordinator's run_jobs already took every job
    while (finished_.load(std::memory_order_acquire) < static_cast<int64_t>(workers_.size()) && !stop_.load(std::memory_order_relaxed))
      cpu_relax();
  }

  void coordinator_loop(int cpu) {
    if (cpu >= 0) pin_this_thread(cpu);
    auto last_active = fetch_clock::now();
    int64_t sleep_us = 20;
    while (!stop_.load(std::memory_order_relaxed)) {
      const uint64_t task = sync_->cpu_poll();
      if (task == 0) {
        if (fetch_clock::now() - last_active < kHotWindow) {
          cpu_relax();
        } else {
          std::this_thread::sleep_for(std::chrono::microseconds(sleep_us));
          sleep_us = std::min<int64_t>(sleep_us * 2, 2000);
        }
        continue;
      }
      copy_layer(static_cast<int>(task - 1));
      // release: the staging stores are visible before the GPU sees done
      sync_->cpu_done();
      last_active = fetch_clock::now();
      sleep_us = 20;
    }
    sync_->cpu_done();  // a replay caught mid-teardown does not stall
  }

  std::vector<std::vector<uintptr_t>> src_;
  std::vector<std::vector<int64_t>> row_bytes_;
  std::vector<uintptr_t> staging_;
  int64_t max_rows_;
  bool memops_;
  std::unique_ptr<GpuSync> sync_;
  void* meta_ = nullptr;
  const volatile int64_t* count_ = nullptr;
  const int32_t* rows_ = nullptr;
  int64_t* count_dev_ = nullptr;
  int32_t* rows_dev_ = nullptr;
  std::vector<CopyJob> jobs_;
  std::atomic<int64_t> next_{0};
  std::atomic<int64_t> finished_{0};
  std::atomic<uint64_t> gen_{0};
  std::atomic<bool> stop_{false};
  std::vector<std::thread> workers_;
  std::thread coord_;
};

}  // namespace

void bind_locked_fetch(pybind11::module_& m) {
  namespace py = pybind11;
  py::class_<LockedFetch>(m, "LockedFetch")
      .def(py::init<std::vector<std::vector<uintptr_t>>, std::vector<std::vector<int64_t>>, std::vector<uintptr_t>, int, std::vector<int>,
                    bool>(),
           py::arg("src"), py::arg("row_bytes"), py::arg("staging"), py::arg("max_rows"), py::arg("cpus"), py::arg("memops"))
      .def("publish", &LockedFetch::publish, py::arg("stream"), py::arg("layer"), py::arg("num_indices"), py::arg("src_indices"))
      .def("done_word", &LockedFetch::done_word);
}

}  // namespace freetoken::cpu_moe