Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

17.2 KB
Raw Blame
//
// MIT license
// Copyright (C) 2024 Intel Corporation
// SPDX-License-Identifier: MIT
//

//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//

#pragma once

#include <array>
#include <cstdlib>
#include <cstdint>
#include <cstdio>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <vector>

#if ARK_DNNL
#include <dnnl.hpp>
#endif

#include "bestla/bestla_wrapper.h"

#if ARK_XPU
#include "bestla/sycl/sycl_wrapper.h"
#else
namespace sycl {
typedef void queue;
}
#endif

#if ARK_XPU && ARK_DNNL
#include <dnnl_sycl.hpp>
#endif

#define LOG_LINE() printf("%s:L%d\n", __FUNCTION__, __LINE__);

namespace ark {

struct env_params {
  int verbose = 2;  // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6
  int auto_s8 = -1;
  int sage_use_mean_bias = 1;
  int sage_print_kbias = 0;
  int sage_disable_packed_hnd_fast = 0;

  static env_params* Instance() {
    static env_params instance;
    return &instance;
  }

  env_params() {
    env_i("ARK_VERBOSE", verbose);
    env_i("ARK_AUTO_S8", auto_s8);
    env_i("ARK_SAGE_USE_MEAN_BIAS", sage_use_mean_bias);
    env_i("ARK_SAGE_PRINT_KBIAS", sage_print_kbias);
    env_i("ARK_SAGE_DISABLE_PACKED_HND_FAST", sage_disable_packed_hnd_fast);
  }

  static inline void env_i(const char* envstr, int& default_) {
    const char* env_value = std::getenv(envstr);
    if (env_value != nullptr) default_ = std::stoi(env_value);
  }
};

using UUIDArray = std::array<unsigned char, 16>;

struct UUIDHasher {
  size_t operator()(const UUIDArray& uuid) const {
    const uint64_t* p = reinterpret_cast<const uint64_t*>(uuid.data());
    uint64_t h1 = p[0];
    uint64_t h2 = p[1];

    h1 ^= h1 >> 33;
    h1 *= 0xff51afd7ed558ccdLLU;
    h2 ^= h2 >> 33;
    h2 *= 0xc4ceb9fe1a85ec53LLU;

    return static_cast<size_t>(h1 ^ h2);
  }
};

class DeviceMemoryPool {
 public:
  static DeviceMemoryPool* Instance() {
    static DeviceMemoryPool instance;
    return &instance;
  }

  size_t get_device_key(sycl::queue* q) {
#if ARK_XPU
    if (q != nullptr) {
      std::lock_guard<std::mutex> lock(key_mutex_);
      auto key = get_device_queue_key_locked(q);
      auto it = device_queue_ids_.find(key);
      if (it != device_queue_ids_.end()) {
        return it->second;
      }
      const size_t new_id = next_device_queue_id_++;
      device_queue_ids_.emplace(key, new_id);
      return new_id;
    }
#endif
    return 0;
  }

  size_t get_device_context_key(sycl::queue* q) {
#if ARK_XPU
    if (q != nullptr) {
      std::lock_guard<std::mutex> lock(key_mutex_);
      auto key = get_device_context_key_locked(q);
      auto it = device_context_ids_.find(key);
      if (it != device_context_ids_.end()) {
        return it->second;
      }
      const size_t new_id = next_device_context_id_++;
      device_context_ids_.emplace(key, new_id);
      return new_id;
    }
#endif
    return 0;
  }

  void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {
    auto key = get_device_key(q);
    return get_scratch_ptr(size, buf_loc, q, key);
  }

  // Current size of the slab held for ``buf_loc`` on ``q``'s device+context+queue, or 0 when
  // no slab is held.
  size_t get_scratch_size(size_t buf_loc, sycl::queue* q) {
    if (buf_loc >= MaxLocNum) return 0;
    auto key = get_device_key(q);
    std::lock_guard<std::mutex> lock(scratch_mutex_);
    auto it = dev_mem_size_map[buf_loc].find(key);
    return it == dev_mem_size_map[buf_loc].end() ? 0 : it->second;
  }

  // Detach the slab held for ``buf_loc`` on ``q``'s device+context+queue and hand ownership to
  // the caller, which becomes responsible for synchronizing and freeing it.
  // Returns nullptr when no slab is held.
  //
  // Detaching rather than freeing in place lets a caller drop the entry while
  // holding its own lock and then perform the (unbounded) `wait()` and
  // `sycl::free()` outside that lock: a concurrent acquire that races in after
  // the detach simply allocates a fresh slab instead of racing on this one.
  void* detach_scratch_mem(size_t buf_loc, sycl::queue* q) {
    if (buf_loc >= MaxLocNum) return nullptr;
    auto key = get_device_key(q);
    std::lock_guard<std::mutex> lock(scratch_mutex_);
    auto it = dev_mem_ptr_map[buf_loc].find(key);
    if (it == dev_mem_ptr_map[buf_loc].end()) return nullptr;
    int8_t* ptr = it->second;
    dev_mem_ptr_map[buf_loc].erase(it);
    dev_mem_size_map[buf_loc].erase(key);
    return ptr;
  }

  void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
    if (size == 0 || buf_loc >= MaxLocNum) return nullptr;
    std::lock_guard<std::mutex> lock(scratch_mutex_);

    auto it = dev_mem_ptr_map[buf_loc].find(key);
    if (it == dev_mem_ptr_map[buf_loc].end()) {
      auto newptr = allocate(size, q);
      dev_mem_size_map[buf_loc][key] = size;
      dev_mem_ptr_map[buf_loc][key] = newptr;
      return newptr;
    }

    auto old_size = dev_mem_size_map[buf_loc][key];
    if (old_size < size) {
#if ARK_XPU
      if (q == nullptr) {
        throw std::invalid_argument("DeviceMemoryPool: XPU grow requires a non-null SYCL queue");
      }
      // Do NOT block the host here with q->wait(). Two reasons:
      //   1) It is redundant for correctness. The scratch slab is keyed
      //      per-queue (see get_device_key), so only work enqueued on this
      //      same queue `q` ever references this slab. Both the sycl::free
      //      in release() below and the sycl::aligned_alloc_device in
      //      allocate() are associated with `q`, and `q` is an in-order
      //      queue (ARK device queues are built in_order; torch.xpu
      //      streams are in-order by default). USM bookkeeping plus the
      //      in-order ordering guarantee the old slab is not handed back
      //      while any in-flight kernel still references it -- the same
      //      guarantee the rest of the codebase relies on (e.g. the async
      //      sycl::free in sycl_tla_moe_prefill_fp8_native).
      //   2) A blocking wait() is illegal while `q` is recording to a
      //      torch.xpu command graph, so a wait here would throw whenever a
      //      scratch slab grows during graph capture (see issue #2206).
#endif
      release(it->second, q);
      auto newptr = allocate(size, q);
      dev_mem_size_map[buf_loc][key] = size;
      dev_mem_ptr_map[buf_loc][key] = newptr;
      return newptr;
    }

    return it->second;
  }

 private:
  // Slots 0-7 are claimed by the dnnl / xpu / sycl-s8 / cpu wrappers and the
  // SDPA kernels; slot 8 is the MoE DPAS grouped-GEMM work-group counter
  // (`moe_dpas_fp8::kAtomicScratchLoc`); slots 9 and 10 are the int4 decode
  // weight-repack and activation-sum buffers
  // (`moe_decode_detail::kInt4RepackScratchLoc` / `kActGroupSumScratchLoc`);
  // slot 11 is the XMX Hadamard staging buffer
  // (`kHadamardStagingScratchLoc` in `ark.cpp`); slot 12 is the SYCL-TLA
  // prefill workspace (`kPrefillScratchLoc` in `sycl_tla_wrapper.hpp`).
  // slots 13 and 14 are the W4A8
  // activation-quantization and expert-map buffers
  // (`moe_w4a8::kW4A8QactScratchLoc` / `kW4A8ExpertMapScratchLoc`).
  static constexpr int MaxLocNum = 15;

  using SizeMap = std::unordered_map<size_t, size_t>;
  using PtrMap = std::unordered_map<size_t, int8_t*>;

#if ARK_XPU
  struct DeviceContextKey {
    UUIDArray device_uuid;
    size_t context_id;

    bool operator==(const DeviceContextKey& other) const {
      return context_id == other.context_id && device_uuid == other.device_uuid;
    }
  };

  struct DeviceContextKeyHasher {
    size_t operator()(const DeviceContextKey& key) const {
      size_t h = UUIDHasher{}(key.device_uuid);
      h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
      return h;
    }
  };

  struct DeviceQueueKey {
    UUIDArray device_uuid;
    size_t context_id;
    size_t queue_id;

    bool operator==(const DeviceQueueKey& other) const {
      return queue_id == other.queue_id && context_id == other.context_id && device_uuid == other.device_uuid;
    }
  };

  struct DeviceQueueKeyHasher {
    size_t operator()(const DeviceQueueKey& key) const {
      size_t h = UUIDHasher{}(key.device_uuid);
      h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
      h ^= std::hash<size_t>{}(key.queue_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
      return h;
    }
  };

  UUIDArray get_device_uuid_locked(sycl::queue* q) {
    return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
  }

  DeviceContextKey get_device_context_key_locked(sycl::queue* q) {
    auto context_id = get_context_id_locked(q->get_context());
    return {get_device_uuid_locked(q), context_id};
  }

  size_t get_context_id_locked(const sycl::context& ctx) {
    for (const auto& entry : context_ids_) {
      if (entry.context == ctx) {
        return entry.context_id;
      }
    }
    const size_t new_id = next_context_id_++;
    context_ids_.push_back({ctx, new_id});
    return new_id;
  }

  struct ContextEntry {
    sycl::context context;
    size_t context_id;
  };

  size_t get_queue_id_locked(const sycl::queue& q) {
    for (const auto& entry : queue_ids_) {
      if (entry.queue == q) {
        return entry.queue_id;
      }
    }
    const size_t new_id = next_queue_id_++;
    queue_ids_.push_back({q, new_id});
    return new_id;
  }

  DeviceQueueKey get_device_queue_key_locked(sycl::queue* q) {
    auto context_id = get_context_id_locked(q->get_context());
    auto queue_id = get_queue_id_locked(*q);
    return {get_device_uuid_locked(q), context_id, queue_id};
  }

  struct QueueEntry {
    sycl::queue queue;
    size_t queue_id;
  };
#endif

  int8_t* allocate(size_t size, sycl::queue* q) {
#if ARK_XPU
    if (q == nullptr) {
      throw std::invalid_argument("DeviceMemoryPool: XPU allocation requires a non-null SYCL queue");
    }
    return sycl::aligned_alloc_device<int8_t>(128, size, *q);
#else
    return static_cast<int8_t*>(std::malloc(size));
#endif
  }

  void release(void* ptr, sycl::queue* q) {
    if (ptr == nullptr) return;
#if ARK_XPU
    if (q == nullptr) {
      throw std::invalid_argument("DeviceMemoryPool: XPU free requires a non-null SYCL queue");
    }
    sycl::free(ptr, *q);
#else
    std::free(ptr);
#endif
  }

  std::array<SizeMap, MaxLocNum> dev_mem_size_map;
  std::array<PtrMap, MaxLocNum> dev_mem_ptr_map;
  std::mutex scratch_mutex_;
#if ARK_XPU
  std::mutex key_mutex_;
  std::vector<ContextEntry> context_ids_;
  std::vector<QueueEntry> queue_ids_;
  std::unordered_map<DeviceContextKey, size_t, DeviceContextKeyHasher> device_context_ids_;
  std::unordered_map<DeviceQueueKey, size_t, DeviceQueueKeyHasher> device_queue_ids_;
  size_t next_context_id_ = 1;
  size_t next_queue_id_ = 1;
  size_t next_device_context_id_ = 1;
  size_t next_device_queue_id_ = 1;
#endif
};

#if ARK_DNNL

template <typename>
struct always_false : std::false_type {};

template <typename T>
static inline constexpr dnnl::memory::data_type to_dt() {
  if constexpr (std::is_same_v<T, float>) {
    return dnnl::memory::data_type::f32;
  } else if constexpr (std::is_same_v<T, bestla::utils::fp16>) {
    return dnnl::memory::data_type::f16;
  } else if constexpr (std::is_same_v<T, int8_t>) {
    return dnnl::memory::data_type::s8;
  } else if constexpr (std::is_same_v<T, uint8_t>) {
    return dnnl::memory::data_type::u8;
  } else if constexpr (std::is_same_v<T, bestla::utils::bf16>) {
    return dnnl::memory::data_type::bf16;
  } else {
    static_assert(sizeof(T) == 0, "unsupported data type for to_dt<T>()");
  }
}

static inline constexpr dnnl::memory::data_type to_dt(BTLA_DTYPE bt) {
  switch (bt) {
    case BTLA_DTYPE::F32:
      return dnnl::memory::data_type::f32;
    case BTLA_DTYPE::F16:
      return dnnl::memory::data_type::f16;
    case BTLA_DTYPE::S8:
      return dnnl::memory::data_type::s8;
    case BTLA_DTYPE::U8:
      return dnnl::memory::data_type::u8;
    case BTLA_DTYPE::BF16:
      return dnnl::memory::data_type::bf16;
    default:
      return dnnl::memory::data_type::f32;
  }
}

class DnnlContext {
 public:
  static DnnlContext* Instance() {
    static DnnlContext instance;
    return &instance;
  }

  dnnl::engine* get_eng(sycl::queue* q) {
    std::lock_guard<std::mutex> lock(mutex_);

    if (q == nullptr) {
      if (!cpu_engine_) {
        cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);
      }
      return cpu_engine_.get();
    }

#if ARK_XPU
    auto key = get_device_context_key_locked(q);
    auto engine_it = xpu_engines_.find(key);
    if (engine_it != xpu_engines_.end()) {
      return engine_it->second.get();
    }
    auto dev = q->get_device();
    auto ctx = q->get_context();
    auto insert_result =
        xpu_engines_.emplace(key, std::make_unique<dnnl::engine>(dnnl::sycl_interop::make_engine(dev, ctx)));
    return insert_result.first->second.get();
#else
    if (!cpu_engine_) {
      cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);
    }
    return cpu_engine_.get();
#endif
  }

  dnnl::stream get_stream(sycl::queue* q) {
    auto* eng = get_eng(q);
    if (q == nullptr) {
      return dnnl::stream(*eng);
    }
#if ARK_XPU
    return dnnl::sycl_interop::make_stream(*eng, *q);
#else
    return dnnl::stream(*eng);
#endif
  }

  dnnl::memory get_scratch_mem(dnnl::memory::desc md, sycl::queue* q) {
    auto key = get_dnnl_key(q);
    auto ptr = DeviceMemoryPool::Instance()->get_scratch_ptr(md.get_size(), 0, q, key);
    return dnnl::memory(md, *get_eng(q), ptr);
  }

  void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {
    auto key = get_dnnl_key(q);
    return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
  }

  void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
    if (q == nullptr) {
      return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
    }
#if ARK_XPU
    std::lock_guard<std::mutex> lock(mutex_);
    auto queue_key = DeviceMemoryPool::Instance()->get_device_key(q);
    auto scratch_key = QueueScratchKey{queue_key, key};
    auto it = dnnl_queue_scratch_ids_.find(scratch_key);
    if (it != dnnl_queue_scratch_ids_.end()) {
      return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, it->second);
    }
    const size_t new_id = next_dnnl_queue_scratch_id_++;
    dnnl_queue_scratch_ids_.emplace(scratch_key, new_id);
    return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, new_id);
#else
    return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
#endif
  }

 private:
  struct DeviceContextKey {
    UUIDArray device_uuid;
    size_t context_id;

    bool operator==(const DeviceContextKey& other) const {
      return context_id == other.context_id && device_uuid == other.device_uuid;
    }
  };

  struct DeviceContextKeyHasher {
    size_t operator()(const DeviceContextKey& key) const {
      size_t h = UUIDHasher{}(key.device_uuid);
      h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
      return h;
    }
  };

  size_t get_dnnl_key(sycl::queue* q) {
    if (q == nullptr) return 0;
#if ARK_XPU
    return DeviceMemoryPool::Instance()->get_device_key(q);
#else
    return 0;
#endif
  }

#if ARK_XPU
  UUIDArray get_device_uuid_locked(sycl::queue* q) {
    return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
  }

  DeviceContextKey get_device_context_key_locked(sycl::queue* q) {
    auto context_id = get_context_id_locked(q->get_context());
    return {get_device_uuid_locked(q), context_id};
  }

  size_t get_context_id_locked(const sycl::context& ctx) {
    for (const auto& entry : context_ids_) {
      if (entry.context == ctx) {
        return entry.context_id;
      }
    }
    const size_t new_id = next_context_id_++;
    context_ids_.push_back({ctx, new_id});
    return new_id;
  }

  struct ContextEntry {
    sycl::context context;
    size_t context_id;
  };

  struct QueueScratchKey {
    size_t queue_key;
    size_t user_key;

    bool operator==(const QueueScratchKey& other) const {
      return queue_key == other.queue_key && user_key == other.user_key;
    }
  };

  struct QueueScratchKeyHasher {
    size_t operator()(const QueueScratchKey& key) const {
      size_t h = std::hash<size_t>{}(key.queue_key);
      h ^= std::hash<size_t>{}(key.user_key) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
      return h;
    }
  };

  std::vector<ContextEntry> context_ids_;
  std::unordered_map<QueueScratchKey, size_t, QueueScratchKeyHasher> dnnl_queue_scratch_ids_;
  std::unordered_map<DeviceContextKey, std::unique_ptr<dnnl::engine>, DeviceContextKeyHasher> xpu_engines_;
  size_t next_dnnl_queue_scratch_id_ = 1;
  size_t next_context_id_ = 1;
#endif

  std::unique_ptr<dnnl::engine> cpu_engine_;
  std::mutex mutex_;
};

#endif  // ARK_DNNL

struct QuantParam {
  int n;
  int k;
  int blocksize;
  BTLA_DTYPE compute_type;
  BTLA_DTYPE weight_type;
  BTLA_DTYPE scale_type;
  bool asym;

  QuantParam(int _n, int _k, int _blocksize, int _ct, int _wt, int _st, bool _asym)
      : n(_n),
        k(_k),
        blocksize(_blocksize),
        compute_type((BTLA_DTYPE)_ct),
        weight_type((BTLA_DTYPE)_wt),
        scale_type((BTLA_DTYPE)_st),
        asym(_asym) {}

  inline int blks() { return k / blocksize; }
};

}  // namespace ark