Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
//==---- util.hpp ---------------------------------*- C++ -*----------------==//
//
// Copyright (C) Intel Corporation
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef __DPCT_UTIL_HPP__
#define __DPCT_UTIL_HPP__
#include <sycl/sycl.hpp>
#include <complex>
#include <type_traits>
#include <cassert>
#include <cstdint>
// TODO: Remove these function definitions once they exist in the DPC++ compiler
#if defined(__SYCL_DEVICE_ONLY__) && defined(__INTEL_LLVM_COMPILER)
template <typename T>
__SYCL_CONVERGENT__ extern SYCL_EXTERNAL __SYCL_EXPORT __attribute__((noduplicate))
T __spirv_GroupNonUniformShuffle(__spv::Scope::Flag, T, unsigned) noexcept;
template <typename T>
__SYCL_CONVERGENT__ extern SYCL_EXTERNAL __SYCL_EXPORT __attribute__((noduplicate))
T __spirv_GroupNonUniformShuffleDown(__spv::Scope::Flag, T, unsigned) noexcept;
template <typename T>
__SYCL_CONVERGENT__ extern SYCL_EXTERNAL __SYCL_EXPORT __attribute__((noduplicate))
T __spirv_GroupNonUniformShuffleUp(__spv::Scope::Flag, T, unsigned) noexcept;
#endif
namespace dpct {
/// dim3 is used to store 3 component dimensions.
class dim3 {
public:
unsigned x, y, z;
constexpr dim3(unsigned x = 1, unsigned y = 1, unsigned z = 1)
: x(x), y(y), z(z) {}
dim3(const sycl::id<3> &r) : dim3(r[2], r[1], r[0]) {}
operator sycl::range<3>() const { return sycl::range<3>(z, y, x); }
};
inline dim3 operator*(const dim3 &a, const dim3 &b) {
return dim3{a.x * b.x, a.y * b.y, a.z * b.z};
}
namespace detail {
template <typename tag, typename T> class generic_error_type {
public:
generic_error_type() = default;
generic_error_type(T value) : value{value} {}
operator T() const { return value; }
private:
T value;
};
} // namespace detail
using err0 = detail::generic_error_type<struct err0_tag, int>;
using err1 = detail::generic_error_type<struct err1_tag, int>;
template <int... Ints> struct integer_sequence {};
template <int Size, int... Ints>
struct make_index_sequence
: public make_index_sequence<Size - 1, Size - 1, Ints...> {};
template <int... Ints>
struct make_index_sequence<0, Ints...> : public integer_sequence<Ints...> {};
template <typename T> struct [[deprecated]] DataType { using T2 = T; };
template <typename T> struct DataType<sycl::vec<T, 2>> {
using T2 = std::complex<T>;
};
[[deprecated("Please use dpct::blas::matrix_mem_copy() instead.")]] inline void
matrix_mem_copy(void *to_ptr, const void *from_ptr, int to_ld, int from_ld,
int rows, int cols, int elem_size,
memcpy_direction direction = automatic,
sycl::queue &queue = dpct::get_default_queue(),
bool async = false) {
if (to_ptr == from_ptr && to_ld == from_ld) {
return;
}
if (to_ld == from_ld) {
size_t copy_size = elem_size * ((cols - 1) * (size_t)to_ld + rows);
if (async)
detail::dpct_memcpy(queue, (void *)to_ptr, (void *)from_ptr, copy_size,
direction);
else
detail::dpct_memcpy(queue, (void *)to_ptr, (void *)from_ptr, copy_size,
direction)
.wait();
} else {
if (async)
detail::dpct_memcpy(queue, to_ptr, from_ptr, elem_size * to_ld,
elem_size * from_ld, elem_size * rows, cols,
direction);
else
sycl::event::wait(detail::dpct_memcpy(
queue, to_ptr, from_ptr, elem_size * to_ld, elem_size * from_ld,
elem_size * rows, cols, direction));
}
}
/// Copy matrix data. The default leading dimension is column.
/// \param [out] to_ptr A pointer points to the destination location.
/// \param [in] from_ptr A pointer points to the source location.
/// \param [in] to_ld The leading dimension the destination matrix.
/// \param [in] from_ld The leading dimension the source matrix.
/// \param [in] rows The number of rows of the source matrix.
/// \param [in] cols The number of columns of the source matrix.
/// \param [in] direction The direction of the data copy.
/// \param [in] queue The queue where the routine should be executed.
/// \param [in] async If this argument is true, the return of the function
/// does NOT guarantee the copy is completed.
template <typename T>
[[deprecated("Please use dpct::blas::matrix_mem_copy() instead.")]] inline void
matrix_mem_copy(T *to_ptr, const T *from_ptr, int to_ld, int from_ld, int rows,
int cols, memcpy_direction direction = automatic,
sycl::queue &queue = dpct::get_default_queue(),
bool async = false) {
using Ty = typename DataType<T>::T2;
matrix_mem_copy((void *)to_ptr, (void *)from_ptr, to_ld, from_ld, rows, cols,
sizeof(Ty), direction, queue, async);
}
/// Cast the high or low 32 bits of a double to an integer.
/// \param [in] d The double value.
/// \param [in] use_high32 Cast the high 32 bits of the double if true;
/// otherwise cast the low 32 bits.
inline int cast_double_to_int(double d, bool use_high32 = true) {
sycl::vec<double, 1> v0{d};
auto v1 = v0.as<sycl::int2>();
if (use_high32)
return v1[1];
return v1[0];
}
/// Combine two integers, the first as the high 32 bits and the second
/// as the low 32 bits, into a double.
/// \param [in] high32 The integer as the high 32 bits
/// \param [in] low32 The integer as the low 32 bits
inline double cast_ints_to_double(int high32, int low32) {
sycl::int2 v0{low32, high32};
auto v1 = v0.as<sycl::vec<double, 1>>();
return v1;
}
/// Reverse the bit order of an unsigned integer
/// \param [in] a Input unsigned integer value
/// \returns Value of a with the bit order reversed
template <typename T> inline T reverse_bits(T a) {
static_assert(std::is_unsigned<T>::value && std::is_integral<T>::value,
"unsigned integer required");
if (!a)
return 0;
T mask = 0;
size_t count = 4 * sizeof(T);
mask = ~mask >> count;
while (count) {
a = ((a & mask) << count) | ((a & ~mask) >> count);
count = count >> 1;
mask = mask ^ (mask << count);
}
return a;
}
/// \param [in] a The first value contains 4 bytes
/// \param [in] b The second value contains 4 bytes
/// \param [in] s The selector value, only lower 16bit used
/// \returns the permutation result of 4 bytes selected in the way
/// specified by \p s from \p a and \p b
inline unsigned int byte_level_permute(unsigned int a, unsigned int b,
unsigned int s) {
unsigned int ret;
ret =
((((std::uint64_t)b << 32 | a) >> (s & 0x7) * 8) & 0xff) |
(((((std::uint64_t)b << 32 | a) >> ((s >> 4) & 0x7) * 8) & 0xff) << 8) |
(((((std::uint64_t)b << 32 | a) >> ((s >> 8) & 0x7) * 8) & 0xff) << 16) |
(((((std::uint64_t)b << 32 | a) >> ((s >> 12) & 0x7) * 8) & 0xff) << 24);
return ret;
}
/// \param [in] low32 The 4 bytes to construct the 8 bytes value as low 32 bits.
/// \param [in] high32 The 4 bytes to construct the 8 bytes value as high 32
/// bits.
/// \param [in] sel The selector value. It is used to generate selectors which are
/// used to fetch byte value from \p low32 and \p high32 to construct result
/// value.
/// \param [in] mode The mode of permutation, together with \p sel, it
/// further defines the behavior of data selection. \p mode and \p sel define 4
/// selectors (s[i], i=0, 1, 2, 3), which are used as index to fetch 4 bytes from
/// the 8 bytes value constructed by \p low32 and \p high32; the byte selected by s[i]
/// is used to fill the i-th byte of the result.
/// The available mode values are:
/// mode value 0: s[i] = sel[i * 4 + 3 : i * 4]
/// mode value 1: s[i] = sel[1 : 0] + i
/// mode value 2: s[i] = (sel[1 : 0] + 7) % 8
/// mode value 3: s[i] = sel[1 : 0]
/// mode value 4: s[i] = max(sel[1 : 0], i)
/// mode value 5: s[i] = min(sel[1 : 0], i)
/// mode value 6: s[0] = sel[0 : 0] * 2
/// s[1] = sel[0 : 0] * 2 + 1
/// s[2] = s[0]
/// s[3] = s[1]
/// other value: illegal, undefined behavior, return 0.
inline uint32_t byte_level_permute_custom(uint32_t low32, uint32_t high32,
uint32_t sel, int mode = 0) {
constexpr uint16_t lookup[6][4] = {
{0x3210, 0x4321, 0x5432, 0x6543}, // Forward 4-byte extract
{0x5670, 0x6701, 0x7012, 0x0123}, // Backward 4-byte extract
{0x0000, 0x1111, 0x2222, 0x3333}, // Replicate 8-bit values
{0x3210, 0x3211, 0x3222, 0x3333}, // Edge clamp left
{0x0000, 0x1110, 0x2210, 0x3210}, // Edge clamp right
{0x1010, 0x3232, 0x1010, 0x3232} // Replicate 16-bit values
};
if (mode >= 1 && mode <= 6) {
return byte_level_permute(low32, high32, lookup[mode - 1][sel & 0x3]);
} else if (!mode) {
return byte_level_permute(low32, high32, sel);
}
return 0;
}
/// Find position of first least significant set bit in an integer.
/// ffs(0) returns 0.
///
/// \param [in] a Input integer value
/// \returns The position
template <typename T> inline int ffs(T a) {
static_assert(std::is_integral<T>::value, "integer required");
return (sycl::ctz(a) + 1) % (sizeof(T) * 8 + 1);
}
/// select_from_sub_group allows work-items to obtain a copy of a value held by
/// any other work-item in the sub_group. The input sub_group will be divided
/// into several logical sub_groups with id range [0, \p logical_sub_group_size
/// - 1]. Each work-item in logical sub_group gets value from another work-item
/// whose id is \p remote_local_id. If \p remote_local_id is outside the
/// logical sub_group id range, \p remote_local_id will modulo with \p
/// logical_sub_group_size. The \p logical_sub_group_size must be a power of 2
/// and not exceed input sub_group size.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] remote_local_id Input source work item id
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T select_from_sub_group(sycl::sub_group g, T x, int remote_local_id,
int logical_sub_group_size = 32) {
unsigned int start_index =
g.get_local_linear_id() / logical_sub_group_size * logical_sub_group_size;
return sycl::select_from_group(
g, x, start_index + remote_local_id % logical_sub_group_size);
}
/// shift_sub_group_left move values held by the work-items in a sub_group
/// directly to another work-item in the sub_group, by shifting values a fixed
/// number of work-items to the left. The input sub_group will be divided into
/// several logical sub_groups with id range [0, \p logical_sub_group_size - 1].
/// Each work-item in logical sub_group gets value from another work-item whose
/// id is caller's id adds \p delta. If calculated id is outside the logical
/// sub_group id range, the work-item will get value from itself. The \p
/// logical_sub_group_size must be a power of 2 and not exceed input sub_group
/// size.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T shift_sub_group_left(sycl::sub_group g, T x, unsigned int delta,
int logical_sub_group_size = 32) {
unsigned int id = g.get_local_linear_id();
unsigned int end_index =
(id / logical_sub_group_size + 1) * logical_sub_group_size;
T result = sycl::shift_group_left(g, x, delta);
if ((id + delta) >= end_index) {
result = x;
}
return result;
}
/// shift_sub_group_right move values held by the work-items in a sub_group
/// directly to another work-item in the sub_group, by shifting values a fixed
/// number of work-items to the right. The input sub_group will be divided into
/// several logical_sub_groups with id range [0, \p logical_sub_group_size - 1].
/// Each work-item in logical_sub_group gets value from another work-item whose
/// id is caller's id subtracts \p delta. If calculated id is outside the
/// logical sub_group id range, the work-item will get value from itself. The \p
/// logical_sub_group_size must be a power of 2 and not exceed input sub_group
/// size.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T shift_sub_group_right(sycl::sub_group g, T x, unsigned int delta,
int logical_sub_group_size = 32) {
unsigned int id = g.get_local_linear_id();
unsigned int start_index =
id / logical_sub_group_size * logical_sub_group_size;
T result = sycl::shift_group_right(g, x, delta);
if ((id - start_index) < delta) {
result = x;
}
return result;
}
/// permute_sub_group_by_xor permutes values by exchanging values held by pairs
/// of work-items identified by computing the bitwise exclusive OR of the
/// work-item id and some fixed mask. The input sub_group will be divided into
/// several logical sub_groups with id range [0, \p logical_sub_group_size - 1].
/// Each work-item in logical sub_group gets value from another work-item whose
/// id is bitwise exclusive OR of the caller's id and \p mask. If calculated id
/// is outside the logical sub_group id range, the work-item will get value from
/// itself. The \p logical_sub_group_size must be a power of 2 and not exceed
/// input sub_group size.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] mask Input mask
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T permute_sub_group_by_xor(sycl::sub_group g, T x, unsigned int mask,
int logical_sub_group_size = 32) {
unsigned int id = g.get_local_linear_id();
unsigned int start_index =
id / logical_sub_group_size * logical_sub_group_size;
unsigned int target_offset = (id % logical_sub_group_size) ^ mask;
return sycl::select_from_group(g, x,
target_offset < logical_sub_group_size
? start_index + target_offset
: id);
}
/// The function match_any_over_sub_group conducts a comparison of values
/// across work-items within a sub-group. match_any_over_sub_group return a mask
/// in which some bits are set to 1, indicating that the \p value provided by
/// the work-item represented by these bits are equal. The n-th bit of mask
/// representing the work-item with id n. The parameter \p member_mask
/// indicating the work-items participating the call.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] member_mask Input mask
/// \param [in] value Input value
/// \returns The result
template <typename T>
unsigned int match_any_over_sub_group(sycl::sub_group g, unsigned member_mask,
T value) {
static_assert(std::is_trivially_copyable_v<T>,
"Value type must be trivially copyable type.");
if (!member_mask) {
return 0;
}
unsigned int id = g.get_local_linear_id();
unsigned int flag = 0, result = 0, reduce_result = 0;
unsigned int bit_index = 0x1 << id;
bool is_participate = member_mask & bit_index;
T broadcast_value = 0;
bool matched = false;
while (flag != member_mask) {
broadcast_value =
sycl::select_from_group(g, value, sycl::ctz((~flag & member_mask)));
reduce_result = sycl::reduce_over_group(
g, is_participate ? (broadcast_value == value ? bit_index : 0) : 0,
sycl::plus<>());
flag |= reduce_result;
matched = reduce_result & bit_index;
result = matched * reduce_result + (1 - matched) * result;
}
return result;
}
/// The function match_all_over_sub_group conducts a comparison of values
/// across work-items within a sub-group. match_all_over_sub_group return \p
/// member_mask and predicate \p pred will be set to 1 if all \p value that
/// provided by each work-item in \p member_mask are equal, otherwise return 0
/// and the predicate \p pred will be set to 0. The n-th bit of \p member_mask
/// representing the work-item with id n. The parameter \p member_mask
/// indicating the work-items participating the call.
/// \tparam T Input value type
/// \param [in] g Input sub_group
/// \param [in] member_mask Input mask
/// \param [in] value Input value
/// \param [out] pred Output predicate
/// \returns The result
template <typename T>
unsigned int match_all_over_sub_group(sycl::sub_group g, unsigned member_mask,
T value, int *pred) {
static_assert(std::is_trivially_copyable_v<T>,
"Value type must be trivially copyable type.");
if (!member_mask) {
return 0;
}
unsigned int id = g.get_local_linear_id();
unsigned int bit_index = 0x1 << id;
bool is_participate = member_mask & bit_index;
T broadcast_value = sycl::select_from_group(g, value, sycl::ctz(member_mask));
unsigned int reduce_result = sycl::reduce_over_group(
g,
(member_mask & bit_index) ? (broadcast_value == value ? bit_index : 0)
: 0,
sycl::plus<>());
bool all_equal = (reduce_result == member_mask);
*pred = is_participate & all_equal;
return all_equal * member_mask;
}
namespace experimental {
#if defined(__NVPTX__) && defined(__SYCL_DEVICE_ONLY__)
#define SHFL_SYNC(RES, MASK, VAL, SHFL_PARAM, C, SHUFFLE_INSTR) \
if constexpr (std::is_same_v<T, double>) { \
int x_a, x_b; \
asm("mov.b64 {%0,%1},%2;" : "=r"(x_a), "=r"(x_b) : "d"(VAL)); \
auto tmp_a = __nvvm_shfl_sync_##SHUFFLE_INSTR(MASK, x_a, SHFL_PARAM, C); \
auto tmp_b = __nvvm_shfl_sync_##SHUFFLE_INSTR(MASK, x_b, SHFL_PARAM, C); \
asm("mov.b64 %0,{%1,%2};" : "=d"(RES) : "r"(tmp_a), "r"(tmp_b)); \
} else if constexpr (std::is_same_v<T, long> || \
std::is_same_v<T, unsigned long>) { \
int x_a, x_b; \
asm("mov.b64 {%0,%1},%2;" : "=r"(x_a), "=r"(x_b) : "l"(VAL)); \
auto tmp_a = __nvvm_shfl_sync_##SHUFFLE_INSTR(MASK, x_a, SHFL_PARAM, C); \
auto tmp_b = __nvvm_shfl_sync_##SHUFFLE_INSTR(MASK, x_b, SHFL_PARAM, C); \
asm("mov.b64 %0,{%1,%2};" : "=l"(RES) : "r"(tmp_a), "r"(tmp_b)); \
} else if constexpr (std::is_same_v<T, sycl::half>) { \
short tmp_b16; \
asm("mov.b16 %0,%1;" : "=h"(tmp_b16) : "h"(VAL)); \
auto tmp_b32 = __nvvm_shfl_sync_##SHUFFLE_INSTR( \
MASK, static_cast<int>(tmp_b16), SHFL_PARAM, C); \
asm("mov.b16 %0,%1;" : "=h"(RES) : "h"(static_cast<short>(tmp_b32))); \
} else if constexpr (std::is_same_v<T, float>) { \
auto tmp_b32 = __nvvm_shfl_sync_##SHUFFLE_INSTR( \
MASK, __nvvm_bitcast_f2i(VAL), SHFL_PARAM, C); \
RES = __nvvm_bitcast_i2f(tmp_b32); \
} else { \
RES = __nvvm_shfl_sync_##SHUFFLE_INSTR(MASK, VAL, SHFL_PARAM, C); \
}
#endif
/// Masked version of select_from_sub_group, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] remote_local_id Input source work item id
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T select_from_sub_group(unsigned int member_mask,
sycl::sub_group g, T x, int remote_local_id,
int logical_sub_group_size = 32) {
#if defined(__SYCL_DEVICE_ONLY__)
#if defined(__SPIR__)
unsigned int start_index =
g.get_local_linear_id() / logical_sub_group_size * logical_sub_group_size;
unsigned logical_remote_id =
start_index + remote_local_id % logical_sub_group_size;
return __spirv_GroupNonUniformShuffle(__spv::Scope::Subgroup, x, logical_remote_id);
#elif defined(__NVPTX__)
T result;
int cVal = ((32 - logical_sub_group_size) << 8) | 31;
SHFL_SYNC(result, member_mask, x, remote_local_id, cVal, idx_i32)
return result;
#endif
#else
(void)g;
(void)x;
(void)remote_local_id;
(void)logical_sub_group_size;
(void)member_mask;
throw sycl::exception(sycl::errc::runtime, "Masked version of select_from_sub_group not "
"supported on host device.");
#endif // __SYCL_DEVICE_ONLY__
}
/// Masked version of shift_sub_group_left, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T shift_sub_group_left(unsigned int member_mask,
sycl::sub_group g, T x, unsigned int delta,
int logical_sub_group_size = 32) {
#if defined(__SYCL_DEVICE_ONLY__)
#if defined(__SPIR__)
unsigned int id = g.get_local_linear_id();
unsigned int end_index =
(id / logical_sub_group_size + 1) * logical_sub_group_size;
T result = __spirv_GroupNonUniformShuffleDown(__spv::Scope::Subgroup, x, delta);
if ((id + delta) >= end_index) {
result = x;
}
return result;
#elif defined(__NVPTX__)
T result;
int cVal = ((32 - logical_sub_group_size) << 8) | 31;
SHFL_SYNC(result, member_mask, x, delta, cVal, down_i32)
return result;
#endif
#else
(void)g;
(void)x;
(void)delta;
(void)logical_sub_group_size;
(void)member_mask;
throw sycl::exception(sycl::errc::runtime, "Masked version of shift_sub_group_left not "
"supported on host device.");
#endif // __SYCL_DEVICE_ONLY__
}
/// Masked version of shift_sub_group_right, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T shift_sub_group_right(unsigned int member_mask,
sycl::sub_group g, T x, unsigned int delta,
int logical_sub_group_size = 32) {
#if defined(__SYCL_DEVICE_ONLY__)
#if defined(__SPIR__)
unsigned int id = g.get_local_linear_id();
unsigned int start_index =
id / logical_sub_group_size * logical_sub_group_size;
T result = __spirv_GroupNonUniformShuffleUp(__spv::Scope::Subgroup, x, delta);
if ((id - start_index) < delta) {
result = x;
}
return result;
#elif defined(__NVPTX__)
T result;
int cVal = ((32 - logical_sub_group_size) << 8);
SHFL_SYNC(result, member_mask, x, delta, cVal, up_i32)
return result;
#endif
#else
(void)g;
(void)x;
(void)delta;
(void)logical_sub_group_size;
(void)member_mask;
throw sycl::exception(sycl::errc::runtime, "Masked version of shift_sub_group_right not "
"supported on host device.");
#endif // __SYCL_DEVICE_ONLY__
}
/// Masked version of shift_sub_group_left, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam LogicSubGroupSize Input logical sub_group size
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] last_item Index of last thread in logical subgroup
/// \returns The result
template <size_t LogicSubGroupSize, typename T>
T shift_sub_group_left(sycl::sub_group sg, T input, int delta, int last_item,
unsigned member_mask) {
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
T result;
constexpr int cVal = ((32 - LogicSubGroupSize) << 8);
SHFL_SYNC(result, member_mask, input, delta, cVal | last_item, down_i32)
return result;
#else
if ((1U << sg.get_local_linear_id()) & member_mask) {
auto inner = sycl::ext::oneapi::experimental::entangle(sg);
sycl::group_barrier(inner);
}
auto partition =
sycl::ext::oneapi::experimental::chunked_partition<LogicSubGroupSize>(
sg);
int id = partition.get_local_linear_id();
T result = sycl::shift_group_left(partition, input, delta);
if ((id + delta) > last_item)
result = input;
return result;
#endif
}
/// Masked version of shift_sub_group_right, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam LogicSubGroupSize Input logical sub_group size
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] delta Input delta
/// \param [in] first_item Index of first lane in logical subgroup
/// \returns The result
template <size_t LogicSubGroupSize, typename T>
T shift_sub_group_right(sycl::sub_group sg, T input, int delta, int first_item,
unsigned member_mask) {
#if defined(__SYCL_DEVICE_ONLY__) && defined(__NVPTX__)
T result;
constexpr int cVal = ((32 - LogicSubGroupSize) << 8);
SHFL_SYNC(result, member_mask, input, delta, cVal | first_item, up_i32)
return result;
#else
if ((1U << sg.get_local_linear_id()) & member_mask) {
auto inner = sycl::ext::oneapi::experimental::entangle(sg);
sycl::group_barrier(inner);
}
auto partition =
sycl::ext::oneapi::experimental::chunked_partition<LogicSubGroupSize>(
sg);
int id = sg.get_local_linear_id();
T result = sycl::shift_group_right(partition, input, delta);
if ((id - first_item) < delta)
result = input;
return result;
#endif
}
/// Masked version of permute_sub_group_by_xor, which execute masked sub-group
/// operation. The parameter member_mask indicating the work-items participating
/// the call. Whether the n-th bit is set to 1 representing whether the
/// work-item with id n is participating the call. All work-items named in
/// member_mask must be executed with the same member_mask, or the result is
/// undefined.
/// \tparam T Input value type
/// \param [in] member_mask Input mask
/// \param [in] g Input sub_group
/// \param [in] x Input value
/// \param [in] mask Input mask
/// \param [in] logical_sub_group_size Input logical sub_group size
/// \returns The result
template <typename T>
T permute_sub_group_by_xor(unsigned int member_mask,
sycl::sub_group g, T x, unsigned int mask,
int logical_sub_group_size = 32) {
#if defined(__SYCL_DEVICE_ONLY__)
#if defined(__SPIR__)
unsigned int id = g.get_local_linear_id();
unsigned int start_index =
id / logical_sub_group_size * logical_sub_group_size;
unsigned int target_offset = (id % logical_sub_group_size) ^ mask;
unsigned logical_remote_id = (target_offset < logical_sub_group_size) ? start_index + target_offset : id;
return __spirv_GroupNonUniformShuffle(__spv::Scope::Subgroup, x, logical_remote_id);
#elif defined(__NVPTX__)
T result;
int cVal = ((32 - logical_sub_group_size) << 8) | 31;
SHFL_SYNC(result, member_mask, x, mask, cVal, bfly_i32)
return result;
#endif
#else
(void)g;
(void)x;
(void)mask;
(void)logical_sub_group_size;
(void)member_mask;
throw sycl::exception(sycl::errc::runtime, "Masked version of permute_sub_group_by_xor not "
"supported on host device.");
#endif // __SYCL_DEVICE_ONLY__
}
#if defined(__NVPTX__)
#undef SHFL_SYNC
#endif
} // namespace experimental
/// Computes the multiplication of two complex numbers.
/// \tparam T Complex element type
/// \param [in] x The first input complex number
/// \param [in] y The second input complex number
/// \returns The result
template <typename T>
sycl::vec<T, 2> cmul(sycl::vec<T, 2> x, sycl::vec<T, 2> y) {
std::complex<T> t1(x[0], x[1]), t2(y[0], y[1]);
t1 = t1 * t2;
return sycl::vec<T, 2>(t1.real(), t1.imag());
}
/// Computes the division of two complex numbers.
/// \tparam T Complex element type
/// \param [in] x The first input complex number
/// \param [in] y The second input complex number
/// \returns The result
template <typename T>
sycl::vec<T, 2> cdiv(sycl::vec<T, 2> x, sycl::vec<T, 2> y) {
std::complex<T> t1(x[0], x[1]), t2(y[0], y[1]);
t1 = t1 / t2;
return sycl::vec<T, 2>(t1.real(), t1.imag());
}
/// Computes the magnitude of a complex number.
/// \tparam T Complex element type
/// \param [in] x The input complex number
/// \returns The result
template <typename T>
T cabs(sycl::vec<T, 2> x) {
std::complex<T> t(x[0], x[1]);
return std::abs(t);
}
/// Computes the complex conjugate of a complex number.
/// \tparam T Complex element type
/// \param [in] x The input complex number
/// \returns The result
template <typename T>
sycl::vec<T, 2> conj(sycl::vec<T, 2> x) {
std::complex<T> t(x[0], x[1]);
t = std::conj(t);
return sycl::vec<T, 2>(t.real(), t.imag());
}
inline int get_sycl_language_version() {
#ifdef SYCL_LANGUAGE_VERSION
return SYCL_LANGUAGE_VERSION;
#else
return 202000;
#endif
}
// Returns the SYCL event status for the SYCL command.
/// \param [in] event_ptr A pointer points to the SYCL event
/// \returns The execution result for the SYCL command
inline int sycl_event_query(sycl::event *event_ptr) {
if (event_ptr->get_info<sycl::info::event::command_execution_status>() ==
sycl::info::event_command_status::complete) {
// The SYCL command has finished running on the SYCL device.
return 0;
} else {
// The SYCL command has not finished running on the SYCL device.
return 1;
}
}
namespace experimental {
/// Synchronize work items from all work groups within a SYCL kernel.
/// \param [in] item: Represents a work group.
/// \param [in] counter: An atomic object defined on a device memory which can
/// be accessed by work items in all work groups. The initial value of the
/// counter should be zero.
/// Note: Please make sure that all the work items of all work groups within
/// a SYCL kernel can be scheduled actively at the same time on a device.
template <int dimensions = 3>
[[deprecated(
"Please use "
"sycl::group_barrier(sycl::ext::oneapi::experimental::root_group G) "
"instead.")]] inline void
nd_range_barrier(
const sycl::nd_item<dimensions> &item,
sycl::atomic_ref<unsigned int,
#ifdef __AMDGPU__
sycl::memory_order::acq_rel,
#else
sycl::memory_order::seq_cst,
#endif
sycl::memory_scope::device,
sycl::access::address_space::global_space> &counter) {
static_assert(dimensions == 3, "dimensions must be 3.");
unsigned int num_groups = item.get_group_range(2) * item.get_group_range(1) *
item.get_group_range(0);
item.barrier();
if (item.get_local_linear_id() == 0) {
unsigned int inc = 1;
unsigned int old_arrive = 0;
bool is_group0 =
(item.get_group(2) + item.get_group(1) + item.get_group(0) == 0);
if (is_group0) {
inc = 0x80000000 - (num_groups - 1);
}
old_arrive = counter.fetch_add(inc);
// Synchronize all the work groups
while (((old_arrive ^ counter.load()) & 0x80000000) == 0)
;
}
item.barrier();
}
/// Synchronize work items from all work groups within a SYCL kernel.
/// \param [in] item: Represents a work group.
/// \param [in] counter: An atomic object defined on a device memory which can
/// be accessed by work items in all work groups. The initial value of the
/// counter should be zero.
/// Note: Please make sure that all the work items of all work groups within
/// a SYCL kernel can be scheduled actively at the same time on a device.
template <>
[[deprecated(
"Please use "
"sycl::group_barrier(sycl::ext::oneapi::experimental::root_group G) "
"instead.")]] inline void
nd_range_barrier(
const sycl::nd_item<1> &item,
sycl::atomic_ref<unsigned int,
#ifdef __AMDGPU__
sycl::memory_order::acq_rel,
#else
sycl::memory_order::seq_cst,
#endif
sycl::memory_scope::device,
sycl::access::address_space::global_space> &counter) {
unsigned int num_groups = item.get_group_range(0);
item.barrier();
if (item.get_local_linear_id() == 0) {
unsigned int inc = 1;
unsigned int old_arrive = 0;
bool is_group0 = (item.get_group(0) == 0);
if (is_group0) {
inc = 0x80000000 - (num_groups - 1);
}
old_arrive = counter.fetch_add(inc);
// Synchronize all the work groups
while (((old_arrive ^ counter.load()) & 0x80000000) == 0)
;
}
item.barrier();
}
/// The logical-group is a logical collection of some work-items within a
/// work-group.
/// Note: Please make sure that the logical-group size is a power of 2 in the
/// range [1, current_sub_group_size].
template <int dimensions = 3> class logical_group {
sycl::nd_item<dimensions> _item;
sycl::group<dimensions> _g;
uint32_t _logical_group_size;
uint32_t _group_linear_range_in_parent;
public:
/// Dividing \p parent_group into several logical-groups.
/// \param [in] item Current work-item.
/// \param [in] parent_group The group to be divided.
/// \param [in] size The logical-group size.
logical_group(sycl::nd_item<dimensions> item,
sycl::group<dimensions> parent_group, uint32_t size)
: _item(item), _g(parent_group), _logical_group_size(size) {
_group_linear_range_in_parent =
(_g.get_local_linear_range() - 1) / _logical_group_size + 1;
}
logical_group(sycl::nd_item<dimensions> item)
: _item(item), _g(item.get_group()) {}
/// Returns the index of the work-item within the logical-group.
uint32_t get_local_linear_id() const {
return _item.get_local_linear_id() % _logical_group_size;
}
/// Returns the index of the logical-group in the parent group.
uint32_t get_group_linear_id() const {
return _item.get_local_linear_id() / _logical_group_size;
}
/// Returns the number of work-items in the logical-group.
uint32_t get_local_linear_range() const {
if (_g.get_local_linear_range() % _logical_group_size == 0) {
return _logical_group_size;
}
uint32_t last_item_group_id =
_g.get_local_linear_range() / _logical_group_size;
uint32_t first_of_last_group = last_item_group_id * _logical_group_size;
if (_item.get_local_linear_id() >= first_of_last_group) {
return _g.get_local_linear_range() - first_of_last_group;
} else {
return _logical_group_size;
}
}
/// Returns the number of logical-group in the parent group.
uint32_t get_group_linear_range() const {
return _group_linear_range_in_parent;
}
};
// The original source of the functions calculate_max_active_wg_per_xecore and
// calculate_max_potential_wg were under the license below:
//
// Copyright (C) Intel Corporation
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
/// This function is used for occupancy calculation, it computes the max active
/// work-group number per Xe-Core. Ref to
/// https://github.com/oneapi-src/oneAPI-samples/tree/master/Tools/GPU-Occupancy-Calculator
/// \param [out] num_wg Active work-group number.
/// \param [in] wg_size Work-group size.
/// \param [in] slm_size Share local memory size.
/// \param [in] sg_size Sub-group size.
/// \param [in] used_barrier Whether barrier is used.
/// \param [in] used_large_grf Whether large General Register File is used.
/// \return If no error, returns 0.
/// If \p wg_size exceeds the max work-group size, the max work-group size will
/// be used instead of \p wg_size and returns -1.
inline int calculate_max_active_wg_per_xecore(int *num_wg, int wg_size,
int slm_size = 0,
int sg_size = 32,
bool used_barrier = false,
bool used_large_grf = false) {
int ret = 0;
const int slm_size_per_xe_core = 64 * 1024;
const int max_barrier_registers = 32;
dpct::device_ext &dev = dpct::get_current_device();
size_t max_wg_size = dev.get_info<sycl::info::device::max_work_group_size>();
if (wg_size > max_wg_size) {
wg_size = max_wg_size;
ret = -1;
}
int num_threads_ss = 56;
int max_num_wg = 56;
if (dev.has(sycl::aspect::ext_intel_gpu_eu_count_per_subslice) &&
dev.has(sycl::aspect::ext_intel_gpu_hw_threads_per_eu)) {
auto eu_count =
dev.get_info<sycl::info::device::ext_intel_gpu_eu_count_per_subslice>();
auto threads_count =
dev.get_info<sycl::ext::intel::info::device::gpu_hw_threads_per_eu>();
num_threads_ss = eu_count * threads_count;
max_num_wg = eu_count * threads_count;
}
if (used_barrier) {
max_num_wg = max_barrier_registers;
}
// Calculate num_wg_slm
int num_wg_slm = 0;
if (slm_size == 0) {
num_wg_slm = max_num_wg;
} else {
num_wg_slm = std::floor((float)slm_size_per_xe_core / slm_size);
}
// Calculate num_wg_threads
if (used_large_grf)
num_threads_ss = num_threads_ss / 2;
int num_threads = std::ceil((float)wg_size / sg_size);
int num_wg_threads = std::floor((float)num_threads_ss / num_threads);
// Calculate num_wg
*num_wg = (std::min)(num_wg_slm, num_wg_threads);
*num_wg = (std::min)(*num_wg, max_num_wg);
return ret;
}
/// This function is used for occupancy calculation, it computes the work-group
/// number and the work-group size which achieves the maximum occupancy of the
/// device potentially. Ref to
/// https://github.com/oneapi-src/oneAPI-samples/tree/master/Tools/GPU-Occupancy-Calculator
/// \param [out] num_wg Work-group number.
/// \param [out] wg_size Work-group size.
/// \param [in] max_ws_size_for_device_code The maximum working work-group size
/// for current device code logic. Zero means no limitation.
/// \param [in] slm_size Share local memory size.
/// \param [in] sg_size Sub-group size.
/// \param [in] used_barrier Whether barrier is used.
/// \param [in] used_large_grf Whether large General Register File is used.
/// \return Returns 0.
inline int calculate_max_potential_wg(int *num_wg, int *wg_size,
int max_ws_size_for_device_code,
int slm_size = 0, int sg_size = 32,
bool used_barrier = false,
bool used_large_grf = false) {
sycl::device &dev = dpct::get_current_device();
size_t max_wg_size = dev.get_info<sycl::info::device::max_work_group_size>();
if (max_ws_size_for_device_code == 0 ||
max_ws_size_for_device_code >= max_wg_size)
*wg_size = (int)max_wg_size;
else
*wg_size = max_ws_size_for_device_code;
calculate_max_active_wg_per_xecore(num_wg, *wg_size, slm_size, sg_size,
used_barrier, used_large_grf);
std::uint32_t num_ss = 1;
if (dev.has(sycl::aspect::ext_intel_gpu_slices) &&
dev.has(sycl::aspect::ext_intel_gpu_subslices_per_slice)) {
num_ss =
dev.get_info<sycl::ext::intel::info::device::gpu_slices>() *
dev.get_info<sycl::ext::intel::info::device::gpu_subslices_per_slice>();
}
num_wg[0] = num_ss * num_wg[0];
return 0;
}
/// Supported group type during migration.
enum class group_type { work_group, sub_group, logical_group, root_group };
/// The group_base will dispatch the function call to the specific interface
/// based on the group type.
template <int dimensions = 3> class group_base {
public:
group_base(sycl::nd_item<dimensions> item)
: nd_item(item), logical_group(item) {}
~group_base() {}
/// Returns the number of work-items in the group.
size_t get_local_linear_range() {
switch (type) {
case group_type::work_group:
return nd_item.get_group().get_local_linear_range();
case group_type::sub_group:
return nd_item.get_sub_group().get_local_linear_range();
case group_type::logical_group:
return logical_group.get_local_linear_range();
default:
return -1; // Unkonwn group type
}
}
/// Returns the index of the work-item within the group.
size_t get_local_linear_id() {
switch (type) {
case group_type::work_group:
return nd_item.get_group().get_local_linear_id();
case group_type::sub_group:
return nd_item.get_sub_group().get_local_linear_id();
case group_type::logical_group:
return logical_group.get_local_linear_id();
default:
return -1; // Unkonwn group type
}
}
/// Wait for all the elements within the group to complete their execution
/// before proceeding.
void barrier() {
switch (type) {
case group_type::work_group:
sycl::group_barrier(nd_item.get_group());
break;
case group_type::sub_group:
case group_type::logical_group:
sycl::group_barrier(nd_item.get_sub_group());
break;
default:
break;
}
}
protected:
logical_group<dimensions> logical_group;
sycl::nd_item<dimensions> nd_item;
group_type type;
};
/// The group class is a container type that can storage supported group_type.
template <typename T, int dimensions = 3>
class group : public group_base<dimensions> {
using group_base<dimensions>::type;
using group_base<dimensions>::logical_group;
public:
group(T g, sycl::nd_item<dimensions> item) : group_base<dimensions>(item) {
if constexpr (std::is_same_v<T, sycl::sub_group>) {
type = group_type::sub_group;
} else if constexpr (std::is_same_v<T, sycl::group<dimensions>>) {
type = group_type::work_group;
} else if constexpr (std::is_same_v<T, dpct::experimental::logical_group<
dimensions>>) {
logical_group = g;
type = group_type::logical_group;
}
}
};
} // namespace experimental
/// If x <= 2, then return a pointer to the deafult queue;
/// otherwise, return x reinterpreted as a dpct::queue_ptr.
inline queue_ptr int_as_queue_ptr(uintptr_t x) {
return x <= 2 ?
&get_default_queue()
: reinterpret_cast<queue_ptr>(x);
}
template <int n_nondefault_params, int n_default_params, typename T>
class args_selector;
/// args_selector is a helper class for extracting arguments from an
/// array of pointers to arguments or buffer of arguments to pass to a
/// kernel function.
///
/// \param R(Ts...) The type of the kernel
/// \param n_nondefault_params The number of nondefault parameters of the kernel
/// (excluding parameters that like sycl::nd_item, etc.)
/// \param n_default_params The number of default parameters of the kernel
///
/// Example usage:
/// With the following kernel:
/// void foo(sycl::float2 *x, int n, sycl::nd_item<3> item_ct1, float f=.1) {}
/// and with the declaration:
/// args_selector<2, 1, decltype(foo)> selector(kernelParams, extra);
/// we have:
/// selector.get<0>() returns a reference to sycl::float*,
/// selector.get<1>() returns a reference to int,
/// selector.get<2>() returns a reference to float
template <int n_nondefault_params, int n_default_params,
typename R, typename... Ts>
class args_selector<n_nondefault_params, n_default_params, R(Ts...)> {
private:
void **kernel_params;
char *args_buffer;
template <int i>
static constexpr int account_for_default_params() {
constexpr int n_total_params = sizeof...(Ts);
if constexpr (i >= n_nondefault_params) {
return n_total_params - n_default_params + (i - n_nondefault_params);
} else {
return i;
}
}
public:
/// Get the type of the ith argument of R(Ts...)
/// \param [in] i Index of parameter to get
/// \returns Type of ith parameter
template <int i>
using arg_type = std::tuple_element_t<account_for_default_params<i>(),
std::tuple<Ts...>>;
static constexpr int params_num = sizeof...(Ts);
private:
template <int i>
static constexpr int get_offset() {
if constexpr (i == 0) {
// we can assume args_buffer is properly aligned to the
// first argument
return 0;
} else {
constexpr int prev_off = get_offset<i-1>();
constexpr int prev_past_end = prev_off + sizeof(arg_type<i-1>);
using T = arg_type<i>;
// is the past-the-end of the i-1st element properly aligned
// with the ith element's alignment?
if constexpr (prev_past_end % alignof(T) == 0) {
return prev_past_end;
}
// otherwise bump prev_past_end to match alignment
else {
return prev_past_end + (alignof(T) - (prev_past_end % alignof(T)));
}
}
}
static char *get_args_buffer(void **extra) {
if (!extra)
return nullptr;
for (; (std::size_t) *extra != 0; ++extra) {
if ((std::size_t) *extra == 1) {
return static_cast<char*>(*(extra+1));
}
}
return nullptr;
}
public:
/// If kernel_params is nonnull, then args_selector will
/// extract arguments from kernel_params. Otherwise, it
/// will extract them from extra.
/// \param [in] kernel_params Array of pointers to arguments
/// a or null pointer.
/// \param [in] extra Array containing pointer to argument buffer.
args_selector(void **kernel_params, void **extra)
: kernel_params(kernel_params),
args_buffer(get_args_buffer(extra))
{}
/// Get a reference to the ith argument extracted from kernel_params
/// or extra.
/// \param [in] i Index of argument to get
/// \returns Reference to the ith argument
template <int i>
arg_type<i> &get() {
if (kernel_params) {
return *static_cast<arg_type<i>*>(kernel_params[i]);
} else {
return *reinterpret_cast<arg_type<i>*>(args_buffer + get_offset<i>());
}
}
};
/// \brief This struct template used to get the type of the N-th argument of a
/// callable type `Func`. It supports both function types (e.g., `void(int,
/// double)`) and callable objects such as lambdas or functors.
///
/// \tparam Func The callable type from which to extract the argument type.
/// \tparam N The index of the argument to retrieve.
///
/// Example:
/// using Func = void(int, double, const char*);
/// static_assert(std::is_same<nth_argument_type<Func, 0>::type, int>::value,
/// "Unexpected type");
template <typename Func, std::size_t N> struct nth_argument_type {
template <typename R, typename... Args>
static auto
helper(R(Args...)) -> std::tuple_element_t<N, std::tuple<Args...>>;
using type = decltype(helper(std::declval<Func>()));
};
/// \brief The function performs bitwise logical operations on three input
/// values of \p a, \p b and \p c based on the specified 8-bit truth table \p
/// lut and return the result
///
/// \param [in] a Input value
/// \param [in] b Input value
/// \param [in] c Input value
/// \param [in] lut truth table for looking up
/// \returns The result
inline uint32_t ternary_logic_op(uint32_t a, uint32_t b, uint32_t c,
uint8_t lut) {
uint32_t result = 0;
switch (lut) {
case 0x0:
result = 0;
break;
case 0x1:
result = ~a & ~b & ~c;
break;
case 0x2:
result = ~a & ~b & c;
case 0x4:
result = ~a & b & ~c;
break;
case 0x8:
result = ~a & b & c;
break;
case 0x10:
result = a & ~b & ~c;
break;
case 0x20:
result = a & ~b & c;
break;
case 0x40:
result = a & b & ~c;
break;
case 0x80:
result = a & b & c;
break;
case 0x1a:
result = (a & b | c) ^ a;
break;
case 0x1e:
result = a ^ (b | c);
break;
case 0x2d:
result = ~a ^ (~b & c);
break;
case 0x78:
result = a ^ (b & c);
break;
case 0x96:
result = a ^ b ^ c;
break;
case 0xb4:
result = a ^ (b & ~c);
break;
case 0xb8:
result = a ^ (b & (c ^ a));
break;
case 0xd2:
result = a ^ (~b & c);
break;
case 0xe8:
result = a & (b | c) | (b & c);
break;
case 0xea:
result = a & b | c;
break;
case 0xfe:
result = a | b | c;
break;
case 0xff:
result = -1;
break;
default: {
if (lut & 0x01)
result |= ~a & ~b & ~c;
if (lut & 0x02)
result |= ~a & ~b & c;
if (lut & 0x04)
result |= ~a & b & ~c;
if (lut & 0x08)
result |= ~a & b & c;
if (lut & 0x10)
result |= a & ~b & ~c;
if (lut & 0x20)
result |= a & ~b & c;
if (lut & 0x40)
result |= a & b & ~c;
if (lut & 0x80)
result |= a & b & c;
break;
}
}
return result;
}
#ifdef _WIN32
#define DPCT_EXPORT __declspec(dllexport)
#else
#define DPCT_EXPORT
#endif
} // namespace dpct
#endif // __DPCT_UTIL_HPP__