Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
//==---- sparse_utils.hpp -------------------------*- C++ -*----------------==//
//
// Copyright (C) Intel Corporation
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
#ifndef __DPCT_SPARSE_UTILS_HPP__
#define __DPCT_SPARSE_UTILS_HPP__
#include "compat_service.hpp"
#include "lib_common_utils.hpp"
namespace dpct::sparse {
class matrix_info {
public:
/// Matrix types are:
/// ge: General matrix
/// sy: Symmetric matrix
/// he: Hermitian matrix
/// tr: Triangular matrix
enum class matrix_type : int { ge = 0, sy, he, tr };
auto get_matrix_type() const { return _matrix_type; }
auto get_diag() const { return _diag; }
auto get_uplo() const { return _uplo; }
auto get_index_base() const { return _index_base; }
void set_matrix_type(matrix_type mt) { _matrix_type = mt; }
void set_diag(oneapi::mkl::diag d) { _diag = d; }
void set_uplo(oneapi::mkl::uplo u) { _uplo = u; }
void set_index_base(oneapi::mkl::index_base ib) { _index_base = ib; }
private:
matrix_type _matrix_type = matrix_type::ge;
oneapi::mkl::diag _diag = oneapi::mkl::diag::nonunit;
oneapi::mkl::uplo _uplo = oneapi::mkl::uplo::upper;
oneapi::mkl::index_base _index_base = oneapi::mkl::index_base::zero;
};
/// Sparse matrix data format
enum matrix_format : int { csr = 1, coo = 2 };
/// Sparse matrix attribute
enum matrix_attribute : int { uplo = 0, diag };
enum class conversion_scope : int { index = 0, index_and_value };
#ifdef __INTEL_MKL__ // The oneMKL Interfaces Project does not support this.
// Forward declaration
namespace detail {
template <typename T> struct optimize_csrsv_impl;
template <typename T> struct optimize_csrsm_impl;
}
/// Saving the optimization information for solving a system of linear
/// equations.
class optimize_info {
public:
/// Constructor
optimize_info() { oneapi::mkl::sparse::init_matrix_handle(&_matrix_handle); }
/// Destructor
~optimize_info() {
oneapi::mkl::sparse::release_matrix_handle(::dpct::cs::get_default_queue(),
&_matrix_handle, _deps)
.wait();
}
/// Add dependency for the destructor.
/// \param [in] e The event which the destructor depends on.
void add_dependency(sycl::event e) { _deps.push_back(e); }
/// Get the internal saved matrix handle.
/// \return Returns the matrix handle.
oneapi::mkl::sparse::matrix_handle_t get_matrix_handle() const noexcept {
return _matrix_handle;
}
#ifdef DPCT_USM_LEVEL_NONE
template <typename T> friend struct detail::optimize_csrsv_impl;
template <typename T> friend struct detail::optimize_csrsm_impl;
#endif
private:
oneapi::mkl::sparse::matrix_handle_t _matrix_handle = nullptr;
std::vector<sycl::event> _deps;
#ifdef DPCT_USM_LEVEL_NONE
static constexpr size_t _max_data_variable_size =
(std::max)({sizeof(sycl::buffer<float>), sizeof(sycl::buffer<double>),
sizeof(sycl::buffer<std::complex<float>>),
sizeof(sycl::buffer<std::complex<double>>)});
using value_buf_t =
std::variant<std::array<std::byte, _max_data_variable_size>,
sycl::buffer<float>, sycl::buffer<double>,
sycl::buffer<std::complex<float>>,
sycl::buffer<std::complex<double>>>;
sycl::buffer<int> _row_ptr_buf = sycl::buffer<int>(0);
sycl::buffer<int> _col_ind_buf = sycl::buffer<int>(0);
value_buf_t _val_buf;
#endif
};
/// Structure for describe a sparse matrix
class sparse_matrix_desc {
public:
/// Constructor
/// \param [out] desc The descriptor to be created
/// \param [in] row_num Number of rows of the sparse matrix.
/// \param [in] col_num Number of colums of the sparse matrix.
/// \param [in] nnz Non-zero elements in the sparse matrix.
/// \param [in] row_ptr An array of length \p row_num + 1. If the \p row_ptr is
/// NULL, the sparse_matrix_desc will allocate internal memory for it. This
/// internal memory can be gotten from get_shadow_row_ptr().
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] value An array containing the non-zero elements of the sparse matrix.
/// \param [in] row_ptr_type Data type of the \p row_ptr .
/// \param [in] col_ind_type Data type of the \p col_ind .
/// \param [in] base Indicates how input arrays are indexed.
/// \param [in] value_type Data type of the \p value .
/// \param [in] data_format The matrix data format.
sparse_matrix_desc(std::int64_t row_num, std::int64_t col_num,
std::int64_t nnz, void *row_ptr, void *col_ind,
void *value, library_data_t row_ptr_type,
library_data_t col_ind_type, oneapi::mkl::index_base base,
library_data_t value_type, matrix_format data_format)
: _row_num(row_num), _col_num(col_num), _nnz(nnz), _row_ptr(row_ptr),
_col_ind(col_ind), _value(value), _row_ptr_type(row_ptr_type),
_col_ind_type(col_ind_type), _base(base), _value_type(value_type),
_data_format(data_format) {
if (_data_format != matrix_format::csr &&
_data_format != matrix_format::coo) {
throw std::runtime_error("the sparse matrix data format is unsupported");
}
oneapi::mkl::sparse::init_matrix_handle(&_matrix_handle);
set_data();
}
/// Destructor
~sparse_matrix_desc() {
oneapi::mkl::sparse::release_matrix_handle(::dpct::cs::get_default_queue(),
&_matrix_handle, _deps)
.wait();
}
/// Add dependency for the destroy method.
/// \param [in] e The event which the destroy method depends on.
void add_dependency(sycl::event e) { _deps.push_back(e); }
/// Get the internal saved matrix handle.
/// \return Returns the matrix handle.
oneapi::mkl::sparse::matrix_handle_t get_matrix_handle() const noexcept {
return _matrix_handle;
}
/// Get the values saved in the descriptor
/// \param [out] row_num Number of rows of the sparse matrix.
/// \param [out] col_num Number of colums of the sparse matrix.
/// \param [out] nnz Non-zero elements in the sparse matrix.
/// \param [out] row_ptr An array of length \p row_num + 1.
/// \param [out] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [out] value An array containing the non-zero elements of the sparse matrix.
/// \param [out] row_ptr_type Data type of the \p row_ptr .
/// \param [out] col_ind_type Data type of the \p col_ind .
/// \param [out] base Indicates how input arrays are indexed.
/// \param [out] value_type Data type of the \p value .
void get_desc(int64_t *row_num, int64_t *col_num, int64_t *nnz,
void **row_ptr, void **col_ind, void **value,
library_data_t *row_ptr_type, library_data_t *col_ind_type,
oneapi::mkl::index_base *base,
library_data_t *value_type) const noexcept {
*row_num = _row_num;
*col_num = _col_num;
*nnz = _nnz;
*row_ptr = _row_ptr;
*col_ind = _col_ind;
*value = _value;
*row_ptr_type = _row_ptr_type;
*col_ind_type = _col_ind_type;
*base = _base;
*value_type = _value_type;
}
/// Get the sparse matrix data format of this descriptor
/// \param [out] format The matrix data format result
void get_format(matrix_format *data_format) const noexcept {
*data_format = _data_format;
}
/// Get the index base of this descriptor
/// \param [out] base The index base result
void get_base(oneapi::mkl::index_base *base) const noexcept { *base = _base; }
/// Get the value pointer of this descriptor
/// \param [out] value The value pointer result
void get_value(void **value) const noexcept { *value = _value; }
/// Set the value pointer of this descriptor
/// \param [in] value The input value pointer
void set_value(void *value) {
if (!value) {
throw std::runtime_error(
"dpct::sparse::sparse_matrix_desc::set_value(): The value "
"pointer is NULL.");
}
if (_value && (_value != value)) {
throw std::runtime_error(
"dpct::sparse::sparse_matrix_desc::set_value(): "
"The _value pointer is not NULL. It cannot be reset.");
}
_value = value;
set_data();
}
/// Get the size of the sparse matrix
/// \param [out] row_num Number of rows of the sparse matrix.
/// \param [out] col_num Number of colums of the sparse matrix.
/// \param [out] nnz Non-zero elements in the sparse matrix.
void get_size(int64_t *row_num, int64_t *col_num,
int64_t *nnz) const noexcept {
*row_num = _row_num;
*col_num = _col_num;
*nnz = _nnz;
}
/// Set the sparse matrix attribute
/// \param [in] attribute The attribute type
/// \param [in] data The attribute value
/// \param [in] data_size The data size of the attribute value
void set_attribute(matrix_attribute attribute, const void *data,
size_t data_size) {
if (attribute == matrix_attribute::diag) {
const oneapi::mkl::diag *diag_ptr =
reinterpret_cast<const oneapi::mkl::diag *>(data);
if (*diag_ptr == oneapi::mkl::diag::unit) {
_diag = oneapi::mkl::diag::unit;
} else if (*diag_ptr == oneapi::mkl::diag::nonunit) {
_diag = oneapi::mkl::diag::nonunit;
} else {
throw std::runtime_error("unsupported diag value");
}
} else if (attribute == matrix_attribute::uplo) {
const oneapi::mkl::uplo *uplo_ptr =
reinterpret_cast<const oneapi::mkl::uplo *>(data);
if (*uplo_ptr == oneapi::mkl::uplo::upper) {
_uplo = oneapi::mkl::uplo::upper;
} else if (*uplo_ptr == oneapi::mkl::uplo::lower) {
_uplo = oneapi::mkl::uplo::lower;
} else {
throw std::runtime_error("unsupported uplo value");
}
} else {
throw std::runtime_error("unsupported attribute");
}
}
/// Get the sparse matrix attribute
/// \param [out] attribute The attribute type
/// \param [out] data The attribute value
/// \param [out] data_size The data size of the attribute value
void get_attribute(matrix_attribute attribute, void *data,
size_t data_size) const {
if (attribute == matrix_attribute::diag) {
oneapi::mkl::diag *diag_ptr = reinterpret_cast<oneapi::mkl::diag *>(data);
if (_diag.has_value()) {
*diag_ptr = _diag.value();
} else {
*diag_ptr = oneapi::mkl::diag::nonunit;
}
} else if (attribute == matrix_attribute::uplo) {
oneapi::mkl::uplo *uplo_ptr = reinterpret_cast<oneapi::mkl::uplo *>(data);
if (_uplo.has_value()) {
*uplo_ptr = _uplo.value();
} else {
*uplo_ptr = oneapi::mkl::uplo::lower;
}
} else {
throw std::runtime_error("unsupported attribute");
}
}
/// Set the pointers for describing the sparse matrix
/// \param [in] row_ptr An array of length \p row_num + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] value An array containing the non-zero elements of the sparse matrix.
void set_pointers(void *row_ptr, void *col_ind, void *value) {
if (!row_ptr) {
throw std::runtime_error(
"dpct::sparse::sparse_matrix_desc::set_pointers(): The "
"row_ptr pointer is NULL.");
}
if (!col_ind) {
throw std::runtime_error(
"dpct::sparse::sparse_matrix_desc::set_pointers(): The "
"col_ind pointer is NULL.");
}
if (_row_ptr && (_row_ptr != row_ptr)) {
throw std::runtime_error("dpct::sparse::sparse_matrix_desc::set_pointers("
"): The _row_ptr pointer is "
"not NULL. It cannot be reset.");
}
if (_col_ind && (_col_ind != col_ind)) {
throw std::runtime_error("dpct::sparse::sparse_matrix_desc::set_pointers("
"): The _col_ind pointer is "
"not NULL. It cannot be reset.");
}
_row_ptr = row_ptr;
_col_ind = col_ind;
// The descriptor will be updated in the set_value function
set_value(value);
}
/// Get the diag attribute
/// \return diag value
std::optional<oneapi::mkl::diag> get_diag() const noexcept { return _diag; }
/// Get the uplo attribute
/// \return uplo value
std::optional<oneapi::mkl::uplo> get_uplo() const noexcept { return _uplo; }
/// Set the number of non-zero elements
/// \param nnz [in] The number of non-zero elements.
void set_nnz(std::int64_t nnz) noexcept { _nnz = nnz; }
/// Get the type of the value pointer.
/// \return The type of the value pointer.
library_data_t get_value_type() const noexcept { return _value_type; }
/// Get the row_ptr.
/// \return The row_ptr.
void *get_row_ptr() const noexcept { return _row_ptr; }
/// If the internal _row_ptr is NULL, the sparse_matrix_desc will allocate
/// internal memory for it in the constructor. The internal memory can be gotten
/// from this interface.
/// \return The shadow row_ptr.
void *get_shadow_row_ptr() const noexcept { return _shadow_row_ptr.get(); }
/// Get the type of the col_ind pointer.
/// \return The type of the col_ind pointer.
library_data_t get_col_ind_type() const noexcept { return _col_ind_type; }
/// Get the row_num.
/// \return The row_num.
std::int64_t get_row_num() const noexcept { return _row_num; }
private:
inline static const std::function<void(void *)> _shadow_row_ptr_deleter =
[](void *ptr) { ::dpct::cs::free(ptr, ::dpct::cs::get_default_queue()); };
template <typename index_t, typename value_t> void set_data() {
void *row_ptr = nullptr;
if (_shadow_row_ptr) {
row_ptr = _shadow_row_ptr.get();
} else if (_row_ptr) {
row_ptr = _row_ptr;
} else {
row_ptr = ::dpct::cs::malloc(sizeof(index_t) * (_row_num + 1),
::dpct::cs::get_default_queue());
_shadow_row_ptr.reset(row_ptr);
}
#ifdef DPCT_USM_LEVEL_NONE
using data_index_t = sycl::buffer<index_t>;
using data_value_t = sycl::buffer<value_t>;
#else
using data_index_t = index_t *;
using data_value_t = value_t *;
#endif
_data_row_ptr = dpct::detail::get_memory<index_t>(row_ptr);
_data_col_ind = dpct::detail::get_memory<index_t>(_col_ind);
_data_value = dpct::detail::get_memory<value_t>(_value);
if (_data_format == matrix_format::csr)
oneapi::mkl::sparse::set_csr_data(
::dpct::cs::get_default_queue(), _matrix_handle, _row_num, _col_num,
_base, std::get<data_index_t>(_data_row_ptr),
std::get<data_index_t>(_data_col_ind),
std::get<data_value_t>(_data_value));
else
oneapi::mkl::sparse::set_coo_data(
::dpct::cs::get_default_queue(), _matrix_handle, _row_num, _col_num,
_nnz, _base, std::get<data_index_t>(_data_row_ptr),
std::get<data_index_t>(_data_col_ind),
std::get<data_value_t>(_data_value));
::dpct::cs::get_default_queue().wait();
}
void set_data() {
std::uint64_t key = dpct::detail::get_type_combination_id(
_row_ptr_type, _col_ind_type, _value_type);
switch (key) {
case dpct::detail::get_type_combination_id(library_data_t::real_int32,
library_data_t::real_int32,
library_data_t::real_float): {
set_data<std::int32_t, float>();
break;
}
case dpct::detail::get_type_combination_id(library_data_t::real_int32,
library_data_t::real_int32,
library_data_t::real_double): {
set_data<std::int32_t, double>();
break;
}
case dpct::detail::get_type_combination_id(library_data_t::real_int32,
library_data_t::real_int32,
library_data_t::complex_float): {
set_data<std::int32_t, std::complex<float>>();
break;
}
case dpct::detail::get_type_combination_id(
library_data_t::real_int32, library_data_t::real_int32,
library_data_t::complex_double): {
set_data<std::int32_t, std::complex<double>>();
break;
}
case dpct::detail::get_type_combination_id(library_data_t::real_int64,
library_data_t::real_int64,
library_data_t::real_float): {
set_data<std::int64_t, float>();
break;
}
case dpct::detail::get_type_combination_id(library_data_t::real_int64,
library_data_t::real_int64,
library_data_t::real_double): {
set_data<std::int64_t, double>();
break;
}
case dpct::detail::get_type_combination_id(library_data_t::real_int64,
library_data_t::real_int64,
library_data_t::complex_float): {
set_data<std::int64_t, std::complex<float>>();
break;
}
case dpct::detail::get_type_combination_id(
library_data_t::real_int64, library_data_t::real_int64,
library_data_t::complex_double): {
set_data<std::int64_t, std::complex<double>>();
break;
}
default:
throw std::runtime_error("the combination of data type is unsupported");
}
}
std::int64_t _row_num;
std::int64_t _col_num;
std::int64_t _nnz;
void *_row_ptr;
void *_col_ind;
void *_value;
library_data_t _row_ptr_type;
library_data_t _col_ind_type;
oneapi::mkl::index_base _base;
library_data_t _value_type;
oneapi::mkl::sparse::matrix_handle_t _matrix_handle = nullptr;
std::vector<sycl::event> _deps;
matrix_format _data_format;
std::optional<oneapi::mkl::uplo> _uplo;
std::optional<oneapi::mkl::diag> _diag;
std::unique_ptr<void, std::function<void(void *)>> _shadow_row_ptr =
std::unique_ptr<void, std::function<void(void *)>>(
nullptr, _shadow_row_ptr_deleter);
static constexpr size_t _max_data_variable_size = (std::max)(
{sizeof(sycl::buffer<std::int32_t>), sizeof(sycl::buffer<std::int64_t>),
sizeof(sycl::buffer<float>), sizeof(sycl::buffer<double>),
sizeof(sycl::buffer<std::complex<float>>),
sizeof(sycl::buffer<std::complex<double>>), sizeof(void *)});
using index_variant_t =
std::variant<std::array<std::byte, _max_data_variable_size>,
sycl::buffer<std::int32_t>, sycl::buffer<std::int64_t>,
std::int32_t *, std::int64_t *>;
using value_variant_t =
std::variant<std::array<std::byte, _max_data_variable_size>,
sycl::buffer<float>, sycl::buffer<double>,
sycl::buffer<std::complex<float>>,
sycl::buffer<std::complex<double>>, float *, double *,
std::complex<float> *, std::complex<double> *>;
index_variant_t _data_row_ptr;
index_variant_t _data_col_ind;
value_variant_t _data_value;
};
#endif
class sparse_matrix_desc;
using sparse_matrix_desc_t = std::shared_ptr<sparse_matrix_desc>;
/// Structure for describe a dense vector
class dense_vector_desc {
public:
dense_vector_desc(std::int64_t ele_num, void *value,
library_data_t value_type)
: _ele_num(ele_num), _value(value), _value_type(value_type) {}
void get_desc(std::int64_t *ele_num, const void **value,
library_data_t *value_type) const noexcept {
*ele_num = _ele_num;
*value = _value;
*value_type = _value_type;
}
void get_desc(std::int64_t *ele_num, void **value,
library_data_t *value_type) const noexcept {
get_desc(ele_num, const_cast<const void **>(value), value_type);
}
void *get_value() const noexcept { return _value; }
void set_value(void *value) { _value = value; }
library_data_t get_value_type() const noexcept { return _value_type; }
std::int64_t get_ele_num() const noexcept { return _ele_num; }
private:
std::int64_t _ele_num;
void *_value;
library_data_t _value_type;
};
/// Structure for describe a dense matrix
class dense_matrix_desc {
public:
dense_matrix_desc(std::int64_t row_num, std::int64_t col_num,
std::int64_t leading_dim, void *value,
library_data_t value_type, oneapi::mkl::layout layout)
: _row_num(row_num), _col_num(col_num), _leading_dim(leading_dim),
_value(value), _value_type(value_type), _layout(layout) {}
void get_desc(std::int64_t *row_num, std::int64_t *col_num,
std::int64_t *leading_dim, void **value,
library_data_t *value_type,
oneapi::mkl::layout *layout) const noexcept {
*row_num = _row_num;
*col_num = _col_num;
*leading_dim = _leading_dim;
*value = _value;
*value_type = _value_type;
*layout = _layout;
}
void *get_value() const noexcept { return _value; }
void set_value(void *value) { _value = value; }
std::int64_t get_col_num() const noexcept { return _col_num; }
std::int64_t get_leading_dim() const noexcept { return _leading_dim; }
oneapi::mkl::layout get_layout() const noexcept { return _layout; }
private:
std::int64_t _row_num;
std::int64_t _col_num;
std::int64_t _leading_dim;
void *_value;
library_data_t _value_type;
oneapi::mkl::layout _layout;
};
} // namespace dpct::sparse
#include "detail/sparse_utils_detail.hpp"
namespace dpct::sparse {
class descriptor {
public:
descriptor() {}
void set_queue(::dpct::cs::queue_ptr q_ptr) noexcept { _queue_ptr = q_ptr; }
sycl::queue &get_queue() const noexcept { return *_queue_ptr; }
private:
#ifdef __INTEL_MKL__
struct matmat_info {
detail::matrix_handle_manager matrix_handle_a;
detail::matrix_handle_manager matrix_handle_b;
detail::matrix_handle_manager matrix_handle_c;
detail::handle_manager<oneapi::mkl::sparse::matmat_descr_t> matmat_desc;
bool is_empty() { return matmat_desc.is_empty(); }
void init(sycl::queue *q_ptr) {
matmat_desc.init(q_ptr);
matrix_handle_a.init(q_ptr);
matrix_handle_b.init(q_ptr);
matrix_handle_c.init(q_ptr);
}
};
std::unordered_map<detail::csrgemm_args_info, matmat_info,
detail::csrgemm_args_info_hash> &
get_csrgemm_info_map() {
return _csrgemm_info_map;
}
std::unordered_map<detail::csrgemm_args_info, matmat_info,
detail::csrgemm_args_info_hash>
_csrgemm_info_map;
template <typename T>
friend void
csrgemm_nnz(descriptor *desc, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const T *val_a, const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const T *val_b, const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_c, int *row_ptr_c,
int *nnz_host_ptr);
template <typename T>
friend void csrgemm(descriptor *desc, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, const T *val_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, const T *val_b,
const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_c, T *val_c,
const int *row_ptr_c, int *col_ind_c);
#endif
::dpct::cs::queue_ptr _queue_ptr = &::dpct::cs::get_default_queue();
};
using descriptor_ptr = descriptor *;
/// Computes a CSR format sparse matrix-dense vector product.
/// y = alpha * op(A) * x + beta * y
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans The operation applied to the matrix A.
/// \param [in] num_rows Number of rows of the matrix A.
/// \param [in] num_cols Number of columns of the matrix A.
/// \param [in] alpha Scaling factor for the matrix A.
/// \param [in] info Matrix info of the matrix A.
/// \param [in] val An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] x Data of the vector x.
/// \param [in] beta Scaling factor for the vector x.
/// \param [in, out] y Data of the vector y.
template <typename T>
void csrmv(sycl::queue &queue, oneapi::mkl::transpose trans, int num_rows,
int num_cols, const T *alpha,
const std::shared_ptr<matrix_info> info, const T *val,
const int *row_ptr, const int *col_ind, const T *x, const T *beta,
T *y) {
detail::csrmv_impl<T>()(queue, trans, num_rows, num_cols, alpha, info, val,
row_ptr, col_ind, x, beta, y);
}
/// Computes a CSR format sparse matrix-dense vector product. y = A * x
///
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] values An array containing the non-zero elements of the matrix.
/// \param [in] row_offsets An array of length \p num_rows + 1.
/// \param [in] column_indices An array containing the column indices in
/// index-based numbering.
/// \param [in] vector_x Data of the vector x.
/// \param [in, out] vector_y Data of the vector y.
/// \param [in] num_rows Number of rows of the matrix A.
/// \param [in] num_cols Number of columns of the matrix A.
template <typename T>
void csrmv(sycl::queue &queue, const T *values, const int *row_offsets,
const int *column_indices, const T *vector_x, T *vector_y,
int num_rows, int num_cols) {
T alpha{1}, beta{0};
auto matrix_info = std::make_shared<dpct::sparse::matrix_info>();
matrix_info->set_index_base(oneapi::mkl::index_base::zero);
matrix_info->set_matrix_type(dpct::sparse::matrix_info::matrix_type::ge);
detail::csrmv_impl<T>()(queue, oneapi::mkl::transpose::nontrans, num_rows,
num_cols, &alpha, matrix_info, values, row_offsets,
column_indices, vector_x, &beta, vector_y);
}
/// Computes a CSR format sparse matrix-dense vector product.
/// y = alpha * op(A) * x + beta * y
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans The operation applied to the matrix A.
/// \param [in] num_rows Number of rows of the matrix A.
/// \param [in] num_cols Number of columns of the matrix A.
/// \param [in] alpha Scaling factor for the matrix A.
/// \param [in] alpha_type Data type of \p alpha .
/// \param [in] info Matrix info of the matrix A.
/// \param [in] val An array containing the non-zero elements of the matrix A.
/// \param [in] val_type Data type of \p val .
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] x Data of the vector x.
/// \param [in] x_type Data type of \p x .
/// \param [in] beta Scaling factor for the vector x.
/// \param [in] beta_type Data type of \p beta .
/// \param [in, out] y Data of the vector y.
/// \param [in] y_type Data type of \p y .
inline void csrmv(sycl::queue &queue, oneapi::mkl::transpose trans,
int num_rows, int num_cols, const void *alpha,
library_data_t alpha_type,
const std::shared_ptr<matrix_info> info, const void *val,
library_data_t val_type, const int *row_ptr,
const int *col_ind, const void *x, library_data_t x_type,
const void *beta, library_data_t beta_type, void *y,
library_data_t y_type) {
detail::spblas_shim<detail::csrmv_impl>(val_type, queue, trans, num_rows,
num_cols, alpha, info, val, row_ptr,
col_ind, x, beta, y);
}
/// Computes a CSR format sparse matrix-dense matrix product.
/// C = alpha * op(A) * op(B) + beta * C
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a The operation applied to the matrix A.
/// \param [in] trans_b The operation applied to the matrix B.
/// \param [in] sparse_rows Number of rows of the matrix A.
/// \param [in] dense_cols Number of columns of the matrix B or C.
/// \param [in] sparse_cols Number of columns of the matrix A.
/// \param [in] alpha Scaling factor for the matrix A.
/// \param [in] info Matrix info of the matrix A.
/// \param [in] val An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] b Data of the matrix B.
/// \param [in] ldb Leading dimension of the matrix B.
/// \param [in] beta Scaling factor for the matrix B.
/// \param [in, out] c Data of the matrix C.
/// \param [in] ldc Leading dimension of the matrix C.
template <typename T>
void csrmm(sycl::queue &queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, int sparse_rows, int dense_cols,
int sparse_cols, const T *alpha,
const std::shared_ptr<matrix_info> info, const T *val,
const int *row_ptr, const int *col_ind, const T *b, int ldb,
const T *beta, T *c, int ldc) {
#ifndef __INTEL_MKL__
throw std::runtime_error("The oneAPI Math Kernel Library (oneMKL) Interfaces "
"Project does not support this API.");
#else
using Ty = typename ::dpct::detail::lib_data_traits_t<T>;
auto alpha_value =
dpct::detail::get_value(reinterpret_cast<const Ty *>(alpha), queue);
auto beta_value =
dpct::detail::get_value(reinterpret_cast<const Ty *>(beta), queue);
oneapi::mkl::sparse::matrix_handle_t *sparse_matrix_handle =
new oneapi::mkl::sparse::matrix_handle_t;
oneapi::mkl::sparse::init_matrix_handle(sparse_matrix_handle);
auto data_row_ptr = dpct::detail::get_memory<int>(row_ptr);
auto data_col_ind = dpct::detail::get_memory<int>(col_ind);
auto data_val = dpct::detail::get_memory<Ty>(val);
oneapi::mkl::sparse::set_csr_data(queue, *sparse_matrix_handle, sparse_rows,
sparse_cols, info->get_index_base(),
data_row_ptr, data_col_ind, data_val);
auto data_b = dpct::detail::get_memory<Ty>(b);
auto data_c = dpct::detail::get_memory<Ty>(c);
sycl::event gemm_event;
switch (info->get_matrix_type()) {
case matrix_info::matrix_type::ge: {
#ifndef DPCT_USM_LEVEL_NONE
gemm_event =
#endif
oneapi::mkl::sparse::gemm(queue, oneapi::mkl::layout::col_major,
trans_a, trans_b, alpha_value,
*sparse_matrix_handle, data_b, dense_cols,
ldb, beta_value, data_c, ldc);
break;
}
default:
throw std::runtime_error(
"the csrmm does not support matrix_info::matrix_type::sy, "
"matrix_info::matrix_type::tr and matrix_info::matrix_type::he");
}
#ifdef DPCT_USM_LEVEL_NONE
queue.wait();
#endif
sycl::event e = oneapi::mkl::sparse::release_matrix_handle(
queue, sparse_matrix_handle, {gemm_event});
queue.submit([&](sycl::handler &cgh) {
cgh.depends_on(e);
cgh.host_task([=] { delete sparse_matrix_handle; });
});
#endif
}
/// Computes a CSR format sparse matrix-dense matrix product.
/// C = alpha * op(A) * B + beta * C
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans The operation applied to the matrix A.
/// \param [in] sparse_rows Number of rows of the matrix A.
/// \param [in] dense_cols Number of columns of the matrix op(B) or C.
/// \param [in] sparse_cols Number of columns of the matrix A.
/// \param [in] alpha Scaling factor for the matrix A.
/// \param [in] info Matrix info of the matrix A.
/// \param [in] val An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in] b Data of the matrix B.
/// \param [in] ldb Leading dimension of the matrix B.
/// \param [in] beta Scaling factor for the matrix B.
/// \param [in, out] c Data of the matrix C.
/// \param [in] ldc Leading dimension of the matrix C.
template <typename T>
void csrmm(sycl::queue &queue, oneapi::mkl::transpose trans, int sparse_rows,
int dense_cols, int sparse_cols, const T *alpha,
const std::shared_ptr<matrix_info> info, const T *val,
const int *row_ptr, const int *col_ind, const T *b, int ldb,
const T *beta, T *c, int ldc) {
csrmm<T>(queue, trans, oneapi::mkl::transpose::nontrans, sparse_rows,
dense_cols, sparse_cols, alpha, info, val, row_ptr, col_ind, b, ldb,
beta, c, ldc);
}
#ifdef __INTEL_MKL__ // The oneMKL Interfaces Project does not support this.
/// Performs internal optimizations for solving a system of linear equations for
/// a CSR format sparse matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans The operation applied to the sparse matrix.
/// \param [in] row_col Number of rows of the sparse matrix.
/// \param [in] info Matrix info of the sparse matrix.
/// \param [in] val An array containing the non-zero elements of the sparse matrix.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [out] optimize_info The result of the optimizations.
template <typename T>
void optimize_csrsv(sycl::queue &queue, oneapi::mkl::transpose trans,
int row_col, const std::shared_ptr<matrix_info> info,
const T *val, const int *row_ptr, const int *col_ind,
std::shared_ptr<optimize_info> optimize_info) {
detail::optimize_csrsv_impl<T>()(queue, trans, row_col, info, val, row_ptr,
col_ind, optimize_info);
}
inline void optimize_csrsv(sycl::queue &queue, oneapi::mkl::transpose trans,
int row_col, const std::shared_ptr<matrix_info> info,
const void *val, library_data_t val_type,
const int *row_ptr, const int *col_ind,
std::shared_ptr<optimize_info> optimize_info) {
detail::spblas_shim<detail::optimize_csrsv_impl>(
val_type, queue, trans, row_col, info, val, row_ptr, col_ind,
optimize_info);
}
template <typename T>
void csrsv(sycl::queue &queue, oneapi::mkl::transpose trans, int row_col,
const T *alpha, const std::shared_ptr<matrix_info> info,
const T *val, const int *row_ptr, const int *col_ind,
std::shared_ptr<optimize_info> optimize_info, const T *x, T *y) {
detail::csrsv_impl<T>()(queue, trans, row_col, alpha, info, val, row_ptr,
col_ind, optimize_info, x, y);
}
inline void csrsv(sycl::queue &queue, oneapi::mkl::transpose trans, int row_col,
const void *alpha, library_data_t alpha_type,
const std::shared_ptr<matrix_info> info, const void *val,
library_data_t val_type, const int *row_ptr,
const int *col_ind,
std::shared_ptr<optimize_info> optimize_info, const void *x,
library_data_t x_type, void *y, library_data_t y_type) {
detail::spblas_shim<detail::csrsv_impl>(val_type, queue, trans, row_col,
alpha, info, val, row_ptr, col_ind,
optimize_info, x, y);
}
/// Performs internal optimizations for dpct::sparse::csrsm by analyzing
/// the provided matrix structure and operation parameters. The matrix A must be
/// a triangular sparse matrix with the CSR format.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] transa The operation applied to A.
/// \param [in] transb The operation applied to B and X.
/// \param [in] row_col Number of rows and columns of A.
/// \param [in] nrhs Number of columns op_b(B).
/// \param [in] info Matrix info of A.
/// \param [in] val An array containing the non-zero elements of A.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [out] optimize_info The result of the optimizations.
template <typename T>
void optimize_csrsm(sycl::queue &queue, oneapi::mkl::transpose transa,
oneapi::mkl::transpose transb, int row_col, int nrhs,
const std::shared_ptr<matrix_info> info, const T *val,
const int *row_ptr, const int *col_ind,
std::shared_ptr<optimize_info> optimize_info) {
detail::optimize_csrsm_impl<T>()(queue, transa, transb, row_col, nrhs, info,
val, row_ptr, col_ind, optimize_info);
}
/// Solves the sparse triangular system op_a(A) * op_b(X) = alpha * op_b(B).
/// A is a sparse triangular matrix with the CSR format of size \p row_col
/// by \p row_col .
/// B is a dense matrix of size \p row_col by \p nrhs ( \p transb is nontrans)
/// or \p nrhs by \p row_col ( \p transb isn't nontrans).
/// X is the solution dense matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] transa The operation applied to A.
/// \param [in] transb The operation applied to B and X.
/// \param [in] row_col Number of rows and columns of A.
/// \param [in] nrhs Number of columns op_b(B).
/// \param [in] alpha Specifies the scalar.
/// \param [in] info Matrix info of A.
/// \param [in] val An array containing the non-zero elements of A.
/// \param [in] row_ptr An array of length \p num_rows + 1.
/// \param [in] col_ind An array containing the column indices in index-based
/// numbering.
/// \param [in, out] b The RHS matrix. It will be overwritten by the X.
/// \param [in] ldb The leading dimension of B and X.
/// \param [in] optimize_info The result of the optimizations.
template <typename T>
void csrsm(sycl::queue &queue, oneapi::mkl::transpose transa,
oneapi::mkl::transpose transb, int row_col, int nrhs, const T *alpha,
const std::shared_ptr<matrix_info> info, const T *val,
const int *row_ptr, const int *col_ind, T *b, int ldb,
std::shared_ptr<optimize_info> optimize_info) {
detail::csrsm_impl<T>()(queue, transa, transb, row_col, nrhs, alpha, info,
val, row_ptr, col_ind, b, ldb, optimize_info);
}
/// Computes a sparse matrix-dense vector product: y = alpha * op(a) * x + beta * y.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans Specifies operation on input matrix.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] x Specifies the dense vector x.
/// \param [in] beta Specifies the scalar beta.
/// \param [in, out] y Specifies the dense vector y.
/// \param [in] data_type Specifies the data type of \param a, \param x and \param y .
inline void spmv(sycl::queue queue, oneapi::mkl::transpose trans,
const void *alpha, sparse_matrix_desc_t a,
std::shared_ptr<dense_vector_desc> x, const void *beta,
std::shared_ptr<dense_vector_desc> y,
library_data_t data_type) {
detail::spblas_shim<detail::spmv_impl>(data_type, queue, trans, alpha, a, x,
beta, y);
}
/// Computes a sparse matrix-dense matrix product: c = alpha * op(a) * op(b) + beta * c.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] trans_b Specifies operation on input matrix b.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] b Specifies the dense matrix b.
/// \param [in] beta Specifies the scalar beta.
/// \param [in, out] c Specifies the dense matrix c.
/// \param [in] data_type Specifies the data type of \param a, \param b and \param c .
inline void spmm(sycl::queue queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, const void *alpha,
sparse_matrix_desc_t a, std::shared_ptr<dense_matrix_desc> b,
const void *beta, std::shared_ptr<dense_matrix_desc> c,
library_data_t data_type) {
if (b->get_layout() != c->get_layout())
throw std::runtime_error("the layout of b and c are different");
detail::spblas_shim<detail::spmm_impl>(data_type, queue, trans_a, trans_b,
alpha, a, b, beta, c);
}
/// Do initial estimation of work and load balancing of computing a sparse
/// matrix-sparse matrix product.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] trans_b Specifies operation on input matrix b.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] b Specifies the sparse matrix b.
/// \param [in] beta Specifies the scalar beta.
/// \param [in, out] c Specifies the sparse matrix c.
/// \param [in] matmat_descr Describes the sparse matrix-sparse matrix operation
/// to be executed.
/// \param [in, out] size_temp_buffer Specifies the size of workspace.
/// \param [in] temp_buffer Specifies the memory of the workspace.
inline void
spgemm_work_estimation(sycl::queue queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, const void *alpha,
sparse_matrix_desc_t a, sparse_matrix_desc_t b,
const void *beta, sparse_matrix_desc_t c,
oneapi::mkl::sparse::matmat_descr_t matmat_descr,
size_t *size_temp_buffer, void *temp_buffer) {
if (temp_buffer) {
detail::temp_memory<std::int64_t, true, size_t> size_memory(
queue, size_temp_buffer);
detail::temp_memory<std::uint8_t, false> work_memory(queue, temp_buffer);
oneapi::mkl::sparse::matmat(
queue, a->get_matrix_handle(), b->get_matrix_handle(),
c->get_matrix_handle(),
oneapi::mkl::sparse::matmat_request::work_estimation, matmat_descr,
size_memory.get_memory_ptr(), work_memory.get_memory_ptr()
#ifndef DPCT_USM_LEVEL_NONE
, {}
#endif
);
} else {
oneapi::mkl::sparse::set_matmat_data(
matmat_descr, oneapi::mkl::sparse::matrix_view_descr::general, trans_a,
oneapi::mkl::sparse::matrix_view_descr::general, trans_b,
oneapi::mkl::sparse::matrix_view_descr::general);
detail::temp_memory<std::int64_t, true, size_t> size_memory(
queue, size_temp_buffer);
oneapi::mkl::sparse::matmat(
queue, a->get_matrix_handle(), b->get_matrix_handle(),
c->get_matrix_handle(),
oneapi::mkl::sparse::matmat_request::get_work_estimation_buf_size,
matmat_descr, size_memory.get_memory_ptr(), nullptr
#ifndef DPCT_USM_LEVEL_NONE
, {}
#endif
);
}
}
/// Do internal products for computing the C matrix of computing a sparse
/// matrix-sparse matrix product.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] trans_b Specifies operation on input matrix b.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] b Specifies the sparse matrix b.
/// \param [in] beta Specifies the scalar beta.
/// \param [in, out] c Specifies the sparse matrix c.
/// \param [in] matmat_descr Describes the sparse matrix-sparse matrix operation
/// to be executed.
/// \param [in, out] size_temp_buffer Specifies the size of workspace.
/// \param [in] temp_buffer Specifies the memory of the workspace.
inline void spgemm_compute(sycl::queue queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, const void *alpha,
sparse_matrix_desc_t a, sparse_matrix_desc_t b,
const void *beta, sparse_matrix_desc_t c,
oneapi::mkl::sparse::matmat_descr_t matmat_descr,
size_t *size_temp_buffer, void *temp_buffer) {
if (temp_buffer) {
std::int64_t nnz_value = 0;
{
detail::temp_memory<std::int64_t, true, size_t> size_memory(
queue, size_temp_buffer);
detail::temp_memory<std::uint8_t, false> work_memory(queue, temp_buffer);
detail::temp_memory<std::int64_t, true, std::int64_t> nnz_memory(
queue, &nnz_value);
oneapi::mkl::sparse::matmat(
queue, a->get_matrix_handle(), b->get_matrix_handle(),
c->get_matrix_handle(), oneapi::mkl::sparse::matmat_request::compute,
matmat_descr, size_memory.get_memory_ptr(),
work_memory.get_memory_ptr()
#ifndef DPCT_USM_LEVEL_NONE
, {}
#endif
);
oneapi::mkl::sparse::matmat(
queue, a->get_matrix_handle(), b->get_matrix_handle(),
c->get_matrix_handle(), oneapi::mkl::sparse::matmat_request::get_nnz,
matmat_descr, nnz_memory.get_memory_ptr(), nullptr
#ifndef DPCT_USM_LEVEL_NONE
, {}
#endif
);
}
c->set_nnz(nnz_value);
} else {
detail::temp_memory<std::int64_t, true, size_t> size_memory(
queue, size_temp_buffer);
oneapi::mkl::sparse::matmat(
queue, a->get_matrix_handle(), b->get_matrix_handle(),
c->get_matrix_handle(),
oneapi::mkl::sparse::matmat_request::get_compute_buf_size, matmat_descr,
size_memory.get_memory_ptr(), nullptr
#ifndef DPCT_USM_LEVEL_NONE
, {}
#endif
);
}
}
/// Do any remaining internal products and accumulation and transfer into final
/// C matrix arrays of computing a sparse matrix-sparse matrix product.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] trans_b Specifies operation on input matrix b.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] b Specifies the sparse matrix b.
/// \param [in] beta Specifies the scalar beta.
/// \param [in, out] c Specifies the sparse matrix c.
/// \param [in] matmat_descr Describes the sparse matrix-sparse matrix operation
/// to be executed.
inline void spgemm_finalize(sycl::queue queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, const void *alpha,
sparse_matrix_desc_t a, sparse_matrix_desc_t b,
const void *beta, sparse_matrix_desc_t c,
oneapi::mkl::sparse::matmat_descr_t matmat_descr) {
oneapi::mkl::sparse::matmat(queue, a->get_matrix_handle(),
b->get_matrix_handle(), c->get_matrix_handle(),
oneapi::mkl::sparse::matmat_request::finalize,
matmat_descr, nullptr, nullptr
#ifdef DPCT_USM_LEVEL_NONE
);
#else
, {}).wait();
#endif
if (c->get_shadow_row_ptr()) {
switch (c->get_col_ind_type()) {
case library_data_t::real_int32: {
::dpct::cs::memcpy(::dpct::cs::get_default_queue(), c->get_row_ptr(),
c->get_shadow_row_ptr(),
sizeof(std::int32_t) * (c->get_row_num() + 1))
.wait();
break;
}
case library_data_t::real_int64: {
::dpct::cs::memcpy(::dpct::cs::get_default_queue(), c->get_row_ptr(),
c->get_shadow_row_ptr(),
sizeof(std::int64_t) * (c->get_row_num() + 1))
.wait();
break;
}
default:
throw std::runtime_error("dpct::sparse::spgemm_finalize(): The data type "
"of the col_ind in matrix c is unsupported.");
}
}
}
/// Performs internal optimizations for spsv by analyzing the provided matrix
/// structure and operation parameters.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] a Specifies the sparse matrix a.
inline void spsv_optimize(sycl::queue queue, oneapi::mkl::transpose trans_a,
sparse_matrix_desc_t a) {
if (!a->get_uplo() || !a->get_diag()) {
throw std::runtime_error(
"dpct::sparse::spsv_optimize(): oneapi::mkl::sparse::optimize_trsv "
"needs uplo and diag attributes to be specified.");
}
oneapi::mkl::sparse::optimize_trsv(
queue, a->get_uplo().value(), oneapi::mkl::transpose::nontrans,
a->get_diag().value(), a->get_matrix_handle());
}
/// Solves a system of linear equations for a sparse matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] x Specifies the dense vector x.
/// \param [in, out] y Specifies the dense vector y.
/// \param [in] data_type Specifies the data type of \param a, \param x and
/// \param y .
inline void spsv(sycl::queue queue, oneapi::mkl::transpose trans_a,
const void *alpha, sparse_matrix_desc_t a,
std::shared_ptr<dense_vector_desc> x,
std::shared_ptr<dense_vector_desc> y,
library_data_t data_type) {
if (!a->get_uplo() || !a->get_diag()) {
throw std::runtime_error(
"dpct::sparse::spsv(): oneapi::mkl::sparse::trsv needs uplo and diag "
"attributes to be specified.");
}
oneapi::mkl::uplo uplo = a->get_uplo().value();
oneapi::mkl::diag diag = a->get_diag().value();
detail::spblas_shim<detail::spsv_impl>(a->get_value_type(), queue, uplo, diag,
trans_a, alpha, a, x, y);
}
/// Performs internal optimizations for spsm by analyzing the provided matrix
/// structure and operation parameters.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] a Specifies the dense matrix b.
/// \param [in] a Specifies the dense matrix c.
inline void spsm_optimize(sycl::queue queue, oneapi::mkl::transpose trans_a,
sparse_matrix_desc_t a,
std::shared_ptr<dense_matrix_desc> b,
std::shared_ptr<dense_matrix_desc> c) {
if (!a->get_uplo() || !a->get_diag()) {
throw std::runtime_error(
"dpct::sparse::spsm_optimize(): oneapi::mkl::sparse::optimize_trsm "
"needs uplo and diag attributes to be specified.");
}
oneapi::mkl::sparse::optimize_trsm(
queue, b->get_layout(), a->get_uplo().value(), trans_a,
a->get_diag().value(), a->get_matrix_handle(), c->get_col_num());
}
/// Solves a system of linear equations with multiple right-hand sides (RHS)
/// for a triangular sparse matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] trans_a Specifies operation on input matrix a.
/// \param [in] trans_b Specifies operation on input matrix b.
/// \param [in] alpha Specifies the scalar alpha.
/// \param [in] a Specifies the sparse matrix a.
/// \param [in] b Specifies the dense matrix b.
/// \param [in, out] c Specifies the dense matrix c.
/// \param [in] data_type Specifies the data type of \param a, \param b and
/// \param c .
inline void spsm(sycl::queue queue, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, const void *alpha,
sparse_matrix_desc_t a, std::shared_ptr<dense_matrix_desc> b,
std::shared_ptr<dense_matrix_desc> c,
library_data_t data_type) {
if (!a->get_uplo() || !a->get_diag()) {
throw std::runtime_error(
"dpct::sparse::spsm(): oneapi::mkl::sparse::trsm needs uplo and diag "
"attributes to be specified.");
}
oneapi::mkl::uplo uplo = a->get_uplo().value();
oneapi::mkl::diag diag = a->get_diag().value();
detail::spblas_shim<detail::spsm_impl>(data_type, queue, trans_a, trans_b,
uplo, diag, alpha, a, b, c);
}
/// Convert a CSR sparse matrix to a CSC sparse matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] m Number of rows of the matrix.
/// \param [in] n Number of columns of the matrix.
/// \param [in] nnz Number of non-zero elements.
/// \param [in] from_val An array containing the non-zero elements of the input
/// matrix.
/// \param [in] from_row_ptr An array of length \p m + 1.
/// \param [in] from_col_ind An array containing the column indices in
/// index-based numbering.
/// \param [out] to_val An array containing the non-zero elements of the output
/// matrix.
/// \param [out] to_col_ptr An array of length \p n + 1.
/// \param [out] to_row_ind An array containing the row indices in index-based
/// numbering.
/// \param [in] range Specifies the conversion scope.
/// \param [in] base Specifies the index base.
template <typename T>
inline void csr2csc(sycl::queue queue, int m, int n, int nnz, const T *from_val,
const int *from_row_ptr, const int *from_col_ind, T *to_val,
int *to_col_ptr, int *to_row_ind, conversion_scope range,
oneapi::mkl::index_base base) {
detail::csr2csc_impl<T>()(queue, m, n, nnz, from_val, from_row_ptr,
from_col_ind, to_val, to_col_ptr, to_row_ind, range,
base);
}
/// Convert a CSR sparse matrix to a CSC sparse matrix.
/// \param [in] queue The queue where the routine should be executed. It must
/// have the in_order property when using the USM mode.
/// \param [in] m Number of rows of the matrix.
/// \param [in] n Number of columns of the matrix.
/// \param [in] nnz Number of non-zero elements.
/// \param [in] from_val An array containing the non-zero elements of the input
/// matrix.
/// \param [in] from_row_ptr An array of length \p m + 1.
/// \param [in] from_col_ind An array containing the column indices in
/// index-based numbering.
/// \param [out] to_val An array containing the non-zero elements of the output
/// matrix.
/// \param [out] to_col_ptr An array of length \p n + 1.
/// \param [out] to_row_ind An array containing the row indices in index-based
/// numbering.
/// \param [in] value_type Data type of \p from_val and \p to_val .
/// \param [in] range Specifies the conversion scope.
/// \param [in] base Specifies the index base.
inline void csr2csc(sycl::queue queue, int m, int n, int nnz,
const void *from_val, const int *from_row_ptr,
const int *from_col_ind, void *to_val, int *to_col_ptr,
int *to_row_ind, library_data_t value_type,
conversion_scope range, oneapi::mkl::index_base base) {
detail::spblas_shim<detail::csr2csc_impl>(
value_type, queue, m, n, nnz, from_val, from_row_ptr, from_col_ind,
to_val, to_col_ptr, to_row_ind, range, base);
}
/// Calculate the non-zero elements number of the result of a
/// sparse matrix (CSR format)-sparse matrix (CSR format) product:
/// C = op(A) * op(B)
/// \param [in] desc The descriptor of this calculation.
/// \param [in] trans_a The operation applied to the matrix A.
/// \param [in] trans_b The operation applied to the matrix B.
/// \param [in] m The rows number of op(A) and C.
/// \param [in] n The columns number of op(B) and C.
/// \param [in] k The columns number of op(A) and rows number of op(B).
/// \param [in] info_a Matrix info of the matrix A.
/// \param [in] nnz_a Non-zero elements number of matrix A.
/// \param [in] val_a An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr_a An array of length row number + 1.
/// \param [in] col_ind_a An array containing the column indices in index-based
/// numbering.
/// \param [in] info_b Matrix info of the matrix B.
/// \param [in] nnz_b Non-zero elements number of matrix B.
/// \param [in] val_b An array containing the non-zero elements of the matrix B.
/// \param [in] row_ptr_b An array of length row number + 1.
/// \param [in] col_ind_b An array containing the column indices in index-based
/// numbering.
/// \param [in] info_c Matrix info of the matrix C.
/// \param [in] row_ptr_c An array of length row number + 1.
/// \param [out] nnz_host_ptr Non-zero elements number of matrix C.
template <typename T>
void csrgemm_nnz(descriptor_ptr desc, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const T *val_a, const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const T *val_b, const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_c, int *row_ptr_c,
int *nnz_host_ptr) {
using Ty = typename ::dpct::detail::lib_data_traits_t<T>;
sycl::queue &queue = desc->get_queue();
detail::csrgemm_args_info args(trans_a, trans_b, m, n, k, info_a, val_a,
row_ptr_a, col_ind_a, info_b, val_b, row_ptr_b,
col_ind_b, info_c, row_ptr_c);
auto &info = desc->get_csrgemm_info_map()[args];
if (info.is_empty()) {
info.init(&queue);
int rows_c = m;
int cols_c = n;
int rows_a = (trans_a == oneapi::mkl::transpose::nontrans) ? m : k;
int cols_a = (trans_a == oneapi::mkl::transpose::nontrans) ? k : m;
int rows_b = (trans_b == oneapi::mkl::transpose::nontrans) ? k : n;
int cols_b = (trans_b == oneapi::mkl::transpose::nontrans) ? n : k;
info.matrix_handle_a.set_matrix_data<Ty>(
rows_a, cols_a, info_a->get_index_base(), row_ptr_a, col_ind_a, val_a);
info.matrix_handle_b.set_matrix_data<Ty>(
rows_b, cols_b, info_b->get_index_base(), row_ptr_b, col_ind_b, val_b);
// In the future, oneMKL will allow nullptr to be passed in for row_ptr_c in
// the initial calls before matmat. But currently, it needs an array of
// length row_number + 1.
info.matrix_handle_c.set_matrix_data<Ty>(
rows_c, cols_c, info_c->get_index_base(), row_ptr_c, nullptr, nullptr);
oneapi::mkl::sparse::set_matmat_data(
info.matmat_desc.get_handle(),
oneapi::mkl::sparse::matrix_view_descr::general, trans_a,
oneapi::mkl::sparse::matrix_view_descr::general, trans_b,
oneapi::mkl::sparse::matrix_view_descr::general);
}
#ifdef DPCT_USM_LEVEL_NONE
#define __MATMAT(STEP, NNZ_C) \
oneapi::mkl::sparse::matmat(queue, info.matrix_handle_a.get_handle(), \
info.matrix_handle_b.get_handle(), \
info.matrix_handle_c.get_handle(), STEP, \
info.matmat_desc.get_handle(), NNZ_C, nullptr)
#else
#define __MATMAT(STEP, NNZ_C) \
oneapi::mkl::sparse::matmat( \
queue, info.matrix_handle_a.get_handle(), \
info.matrix_handle_b.get_handle(), info.matrix_handle_c.get_handle(), \
STEP, info.matmat_desc.get_handle(), NNZ_C, nullptr, {})
#endif
__MATMAT(oneapi::mkl::sparse::matmat_request::work_estimation, nullptr);
queue.wait();
__MATMAT(oneapi::mkl::sparse::matmat_request::compute, nullptr);
int nnz_c_int = 0;
#ifdef DPCT_USM_LEVEL_NONE
sycl::buffer<std::int64_t, 1> nnz_buf_c(1);
__MATMAT(oneapi::mkl::sparse::matmat_request::get_nnz, &nnz_buf_c);
nnz_c_int = nnz_buf_c.get_host_access(sycl::read_only)[0];
#else
std::int64_t *nnz_c = sycl::malloc_host<std::int64_t>(1, queue);
__MATMAT(oneapi::mkl::sparse::matmat_request::get_nnz, nnz_c);
queue.wait();
nnz_c_int = *nnz_c;
#endif
if (nnz_host_ptr) {
*nnz_host_ptr = nnz_c_int;
}
}
/// Computes a sparse matrix (CSR format)-sparse matrix (CSR format) product:
/// C = op(A) * op(B)
/// \param [in] desc The descriptor of this calculation.
/// \param [in] trans_a The operation applied to the matrix A.
/// \param [in] trans_b The operation applied to the matrix B.
/// \param [in] m The rows number of op(A) and C.
/// \param [in] n The columns number of op(B) and C.
/// \param [in] k The columns number of op(A) and rows number of op(B).
/// \param [in] info_a Matrix info of the matrix A.
/// \param [in] val_a An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr_a An array of length row number + 1.
/// \param [in] col_ind_a An array containing the column indices in index-based
/// numbering.
/// \param [in] info_b Matrix info of the matrix B.
/// \param [in] val_b An array containing the non-zero elements of the matrix B.
/// \param [in] row_ptr_b An array of length row number + 1.
/// \param [in] col_ind_b An array containing the column indices in index-based
/// numbering.
/// \param [in] info_c Matrix info of the matrix C.
/// \param [out] val_c An array containing the non-zero elements of the matrix
/// C.
/// \param [in] row_ptr_c An array of length row number + 1.
/// \param [out] col_ind_c An array containing the column indices in index-based
/// numbering.
template <typename T>
void csrgemm(descriptor_ptr desc, oneapi::mkl::transpose trans_a,
oneapi::mkl::transpose trans_b, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, const T *val_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, const T *val_b,
const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_c, T *val_c,
const int *row_ptr_c, int *col_ind_c) {
using Ty = typename ::dpct::detail::lib_data_traits_t<T>;
sycl::queue &queue = desc->get_queue();
detail::csrgemm_args_info args(trans_a, trans_b, m, n, k, info_a, val_a,
row_ptr_a, col_ind_a, info_b, val_b, row_ptr_b,
col_ind_b, info_c, row_ptr_c);
auto &info = desc->get_csrgemm_info_map()[args];
if (info.is_empty()) {
throw std::runtime_error("csrgemm_nnz is not invoked previously.");
}
int rows_c = m;
int cols_c = n;
info.matrix_handle_c.set_matrix_data<Ty>(
rows_c, cols_c, info_c->get_index_base(), row_ptr_c, col_ind_c, val_c);
sycl::event e;
#ifndef DPCT_USM_LEVEL_NONE
e =
#endif
__MATMAT(oneapi::mkl::sparse::matmat_request::finalize, nullptr);
#undef __MATMAT
info.matrix_handle_a.add_dependency(e);
info.matrix_handle_b.add_dependency(e);
info.matrix_handle_c.add_dependency(e);
info.matmat_desc.add_dependency(e);
desc->get_csrgemm_info_map().erase(args);
}
/// Contains internal information for csrgemm2 by analyzing the provided
/// matrix structure and operation parameters.
class csrgemm2_info {
detail::matrix_handle_manager matrix_handle_a;
detail::matrix_handle_manager matrix_handle_b;
detail::matrix_handle_manager matrix_handle_axb;
detail::matrix_handle_manager matrix_handle_d;
detail::matrix_handle_manager matrix_handle_c;
detail::handle_manager<oneapi::mkl::sparse::matmat_descr_t> matmat_desc;
detail::handle_manager<oneapi::mkl::sparse::omatadd_descr_t> omatadd_desc;
void *row_ptr_axb = nullptr;
void *col_ind_axb = nullptr;
void *val_axb = nullptr;
#ifdef DPCT_USM_LEVEL_NONE
sycl::buffer<std::int64_t, 1> *temp_buffer_1_size;
sycl::buffer<std::int64_t, 1> *temp_buffer_2_size;
sycl::buffer<std::uint8_t, 1> *temp_buffer_1;
sycl::buffer<std::uint8_t, 1> *temp_buffer_2;
#else
std::int64_t *temp_buffer_1_size;
std::int64_t *temp_buffer_2_size;
std::uint8_t *temp_buffer_1;
std::uint8_t *temp_buffer_2;
#endif
enum matrix_c_datatype_t {
mcd_float,
mcd_double,
mcd_float2,
mcd_double2,
};
template <typename T> void set_matrix_c_datatype() {
if constexpr (std::is_same_v<T, float>) {
matrix_c_datatype = matrix_c_datatype_t::mcd_float;
} else if constexpr (std::is_same_v<T, double>) {
matrix_c_datatype = matrix_c_datatype_t::mcd_double;
} else if constexpr (std::is_same_v<T, std::complex<float>>) {
matrix_c_datatype = matrix_c_datatype_t::mcd_float2;
} else {
matrix_c_datatype = matrix_c_datatype_t::mcd_double2;
}
}
matrix_c_datatype_t matrix_c_datatype;
void init(sycl::queue *q_ptr) {
matrix_handle_a.init(q_ptr);
matrix_handle_b.init(q_ptr);
matrix_handle_axb.init(q_ptr);
matrix_handle_d.init(q_ptr);
matrix_handle_c.init(q_ptr);
matmat_desc.init(q_ptr);
omatadd_desc.init(q_ptr);
}
sycl::event release(sycl::queue q, sycl::event dep) {
matrix_handle_a.add_dependency(dep);
matrix_handle_b.add_dependency(dep);
matrix_handle_axb.add_dependency(dep);
matrix_handle_d.add_dependency(dep);
matrix_handle_c.add_dependency(dep);
matmat_desc.add_dependency(dep);
omatadd_desc.add_dependency(dep);
std::vector<sycl::event> events;
events.push_back(matrix_handle_a.release());
events.push_back(matrix_handle_b.release());
events.push_back(matrix_handle_axb.release());
events.push_back(matrix_handle_d.release());
events.push_back(matrix_handle_c.release());
events.push_back(matmat_desc.release());
events.push_back(omatadd_desc.release());
return q.single_task(events, [] {});
}
template <typename T>
friend void csrgemm2_get_buffer_size(
descriptor_ptr desc, int m, int n, int k, const T *alpha,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const int *row_ptr_b, const int *col_ind_b, const T *beta,
const std::shared_ptr<matrix_info> info_d, int nnz_d,
const int *row_ptr_d, const int *col_ind_d,
std::shared_ptr<csrgemm2_info> info, size_t *buffer_size_in_bytes);
friend void csrgemm2_nnz(descriptor_ptr desc, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_d, int nnz_d,
const int *row_ptr_d, const int *col_ind_d,
const std::shared_ptr<matrix_info> info_c,
int *row_ptr_c, int *nnz_ptr,
std::shared_ptr<csrgemm2_info> info, void *buffer);
template <typename T>
friend void
csrgemm2(descriptor_ptr desc, int m, int n, int k, const T *alpha,
const std::shared_ptr<matrix_info> info_a, int nnz_a, const T *val_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b, const T *val_b,
const int *row_ptr_b, const int *col_ind_b, const T *beta,
const std::shared_ptr<matrix_info> info_d, int nnz_d, const T *val_d,
const int *row_ptr_d, const int *col_ind_d,
const std::shared_ptr<matrix_info> info_c, T *val_c,
const int *row_ptr_c, int *col_ind_c,
std::shared_ptr<csrgemm2_info> info, void *buffer);
};
/// Calculate the required workspace size of the following operation:
/// C = alpha * A * B + beta * D
/// \param [in] desc The descriptor of this calculation.
/// \param [in] m The rows number of A, D and C.
/// \param [in] n The columns number of B, D and C.
/// \param [in] k The columns number of A and rows number of B.
/// \param [in] alpha Scaling factor.
/// \param [in] info_a Matrix info of the matrix A.
/// \param [in] nnz_a Non-zero elements number of matrix A.
/// \param [in] row_ptr_a An array of length \p m + 1.
/// \param [in] col_ind_a An array containing the column indices in index-based
/// numbering.
/// \param [in] info_b Matrix info of the matrix B.
/// \param [in] nnz_b Non-zero elements number of matrix B.
/// \param [in] row_ptr_b An array of length \p k + 1.
/// \param [in] col_ind_b An array containing the column indices in index-based
/// numbering.
/// \param [in] beta Scaling factor.
/// \param [in] info_d Matrix info of the matrix D.
/// \param [in] nnz_d Non-zero elements number of matrix D.
/// \param [in] row_ptr_d An array of length \p m + 1.
/// \param [in] col_ind_d An array containing the column indices in index-based
/// numbering.
/// \param [in, out] info The information of csrgemm2 operation.
/// \param [out] buffer_size_in_bytes Workspace memory size in bytes.
template <typename T>
void csrgemm2_get_buffer_size(
descriptor_ptr desc, int m, int n, int k, const T *alpha,
const std::shared_ptr<matrix_info> info_a, int nnz_a, const int *row_ptr_a,
const int *col_ind_a, const std::shared_ptr<matrix_info> info_b, int nnz_b,
const int *row_ptr_b, const int *col_ind_b, const T *beta,
const std::shared_ptr<matrix_info> info_d, int nnz_d, const int *row_ptr_d,
const int *col_ind_d, std::shared_ptr<csrgemm2_info> info,
size_t *buffer_size_in_bytes) {
using Ty = typename ::dpct::detail::lib_data_traits_t<T>;
sycl::queue &queue = desc->get_queue();
info->set_matrix_c_datatype<Ty>();
info->row_ptr_axb = (int *)::dpct::cs::malloc((m + 1) * sizeof(int), queue);
info->init(&queue);
info->matrix_handle_a.set_matrix_data<Ty>(m, k, info_a->get_index_base(),
row_ptr_a, col_ind_a, nullptr);
info->matrix_handle_b.set_matrix_data<Ty>(k, n, info_b->get_index_base(),
row_ptr_b, col_ind_b, nullptr);
info->matrix_handle_axb.set_matrix_data<Ty>(
m, n, oneapi::mkl::index_base::zero, info->row_ptr_axb, nullptr, nullptr);
info->matrix_handle_d.set_matrix_data<Ty>(m, n, info_d->get_index_base(),
row_ptr_d, col_ind_d, nullptr);
info->matrix_handle_c.set_matrix_data<Ty>(m, n, oneapi::mkl::index_base::zero,
nullptr, nullptr, nullptr);
oneapi::mkl::sparse::set_matmat_data(
info->matmat_desc.get_handle(),
oneapi::mkl::sparse::matrix_view_descr::general,
oneapi::mkl::transpose::nontrans,
oneapi::mkl::sparse::matrix_view_descr::general,
oneapi::mkl::transpose::nontrans,
oneapi::mkl::sparse::matrix_view_descr::general);
#ifdef DPCT_USM_LEVEL_NONE
#define __MATMAT(STEP, PTR1, PTR2) \
oneapi::mkl::sparse::matmat(queue, info->matrix_handle_a.get_handle(), \
info->matrix_handle_b.get_handle(), \
info->matrix_handle_axb.get_handle(), \
oneapi::mkl::sparse::matmat_request::STEP, \
info->matmat_desc.get_handle(), PTR1, PTR2)
#else
#define __MATMAT(STEP, PTR1, PTR2) \
oneapi::mkl::sparse::matmat(queue, info->matrix_handle_a.get_handle(), \
info->matrix_handle_b.get_handle(), \
info->matrix_handle_axb.get_handle(), \
oneapi::mkl::sparse::matmat_request::STEP, \
info->matmat_desc.get_handle(), PTR1, PTR2, {})
#endif
std::int64_t nnz_axb = 0;
#ifdef DPCT_USM_LEVEL_NONE
info->temp_buffer_1_size = new sycl::buffer<std::int64_t, 1>(1);
__MATMAT(get_work_estimation_buf_size, info->temp_buffer_1_size, nullptr);
info->temp_buffer_1 = new sycl::buffer<std::uint8_t, 1>(
info->temp_buffer_1_size->get_host_access(sycl::read_only)[0]);
__MATMAT(work_estimation, info->temp_buffer_1_size, info->temp_buffer_1);
info->temp_buffer_2_size = new sycl::buffer<std::int64_t, 1>(1);
__MATMAT(get_compute_structure_buf_size, info->temp_buffer_2_size, nullptr);
info->temp_buffer_2 = new sycl::buffer<std::uint8_t, 1>(
info->temp_buffer_2_size->get_host_access(sycl::read_only)[0]);
__MATMAT(compute_structure, info->temp_buffer_2_size, info->temp_buffer_2);
sycl::buffer<std::int64_t, 1> nnz_axb_buf(1);
__MATMAT(get_nnz, &nnz_axb_buf, nullptr);
nnz_axb = nnz_axb_buf.get_host_access(sycl::read_only)[0];
#else
info->temp_buffer_1_size = sycl::malloc_host<std::int64_t>(1, queue);
__MATMAT(get_work_estimation_buf_size, info->temp_buffer_1_size, nullptr);
queue.wait();
info->temp_buffer_1 =
sycl::malloc_device<std::uint8_t>(info->temp_buffer_1_size[0], queue);
__MATMAT(work_estimation, info->temp_buffer_1_size, info->temp_buffer_1);
info->temp_buffer_2_size = sycl::malloc_host<std::int64_t>(1, queue);
__MATMAT(get_compute_structure_buf_size, info->temp_buffer_2_size, nullptr);
queue.wait();
info->temp_buffer_2 =
sycl::malloc_device<std::uint8_t>(info->temp_buffer_2_size[0], queue);
__MATMAT(compute_structure, info->temp_buffer_2_size, info->temp_buffer_2);
std::int64_t *nnz_axb_ptr = sycl::malloc_host<std::int64_t>(1, queue);
__MATMAT(get_nnz, nnz_axb_ptr, nullptr);
queue.wait();
nnz_axb = *nnz_axb_ptr;
sycl::free(nnz_axb_ptr, queue);
#endif
info->col_ind_axb = (int *)::dpct::cs::malloc(nnz_axb * sizeof(int), queue);
info->val_axb = (Ty *)::dpct::cs::malloc(nnz_axb * sizeof(Ty), queue);
info->matrix_handle_axb.set_matrix_data<Ty>(
m, n, oneapi::mkl::index_base::zero, info->row_ptr_axb, info->col_ind_axb,
info->val_axb);
__MATMAT(finalize_structure, nullptr, nullptr);
std::int64_t ws_size = 0;
oneapi::mkl::sparse::omatadd_buffer_size(
queue, oneapi::mkl::transpose::nontrans, oneapi::mkl::transpose::nontrans,
info->matrix_handle_axb.get_handle(), info->matrix_handle_d.get_handle(),
info->matrix_handle_c.get_handle(),
oneapi::mkl::sparse::omatadd_alg::default_alg,
info->omatadd_desc.get_handle(), ws_size);
*buffer_size_in_bytes = ws_size;
}
/// Calculate the non-zero elements number of the matrix C in following operation:
/// C = alpha * A * B + beta * D
/// \param [in] desc The descriptor of this calculation.
/// \param [in] m The rows number of A, D and C.
/// \param [in] n The columns number of B, D and C.
/// \param [in] k The columns number of A and rows number of B.
/// \param [in] info_a Matrix info of the matrix A.
/// \param [in] nnz_a Non-zero elements number of matrix A.
/// \param [in] row_ptr_a An array of length \p m + 1.
/// \param [in] col_ind_a An array containing the column indices in index-based
/// numbering.
/// \param [in] info_b Matrix info of the matrix B.
/// \param [in] nnz_b Non-zero elements number of matrix B.
/// \param [in] row_ptr_b An array of length \p k + 1.
/// \param [in] col_ind_b An array containing the column indices in index-based
/// numbering.
/// \param [in] info_d Matrix info of the matrix D.
/// \param [in] nnz_d Non-zero elements number of matrix D.
/// \param [in] row_ptr_d An array of length \p m + 1.
/// \param [in] col_ind_d An array containing the column indices in index-based
/// numbering.
/// \param [in] info_c Matrix info of the matrix C.
/// \param [in] row_ptr_c An array of length \p m + 1.
/// \param [out] nnz_ptr Non-zero elements number of matrix C.
/// \param [in] info The information of csrgemm2 operation.
/// \param [in] buffer Workspace memory.
inline void csrgemm2_nnz(descriptor_ptr desc, int m, int n, int k,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const int *row_ptr_b, const int *col_ind_b,
const std::shared_ptr<matrix_info> info_d, int nnz_d,
const int *row_ptr_d, const int *col_ind_d,
const std::shared_ptr<matrix_info> info_c,
int *row_ptr_c, int *nnz_ptr,
std::shared_ptr<csrgemm2_info> info, void *buffer) {
sycl::queue &queue = desc->get_queue();
if (info->matrix_c_datatype ==
csrgemm2_info::matrix_c_datatype_t::mcd_float) {
info->matrix_handle_c.set_matrix_data<float>(m, n, info_c->get_index_base(),
nullptr, nullptr, nullptr);
} else if (info->matrix_c_datatype ==
csrgemm2_info::matrix_c_datatype_t::mcd_double) {
info->matrix_handle_c.set_matrix_data<double>(
m, n, info_c->get_index_base(), nullptr, nullptr, nullptr);
} else if (info->matrix_c_datatype ==
csrgemm2_info::matrix_c_datatype_t::mcd_float2) {
info->matrix_handle_c.set_matrix_data<std::complex<float>>(
m, n, info_c->get_index_base(), nullptr, nullptr, nullptr);
} else {
info->matrix_handle_c.set_matrix_data<std::complex<double>>(
m, n, info_c->get_index_base(), nullptr, nullptr, nullptr);
}
auto data_buffer = dpct::detail::get_memory<std::uint8_t>(buffer);
oneapi::mkl::sparse::omatadd_analyze(
queue, oneapi::mkl::transpose::nontrans, oneapi::mkl::transpose::nontrans,
info->matrix_handle_axb.get_handle(), info->matrix_handle_d.get_handle(),
info->matrix_handle_c.get_handle(),
oneapi::mkl::sparse::omatadd_alg::default_alg,
info->omatadd_desc.get_handle(),
#ifdef DPCT_USM_LEVEL_NONE
&data_buffer
#else
data_buffer
#endif
);
std::int64_t nnz_c = 0;
oneapi::mkl::sparse::omatadd_get_nnz(
queue, oneapi::mkl::transpose::nontrans, oneapi::mkl::transpose::nontrans,
info->matrix_handle_axb.get_handle(), info->matrix_handle_d.get_handle(),
info->matrix_handle_c.get_handle(),
oneapi::mkl::sparse::omatadd_alg::default_alg,
info->omatadd_desc.get_handle(), nnz_c);
assert((nnz_c >= INT_MIN && nnz_c <= INT_MAX) && "nnz_c is out of range.");
int nnz_c_int = nnz_c;
if (nnz_ptr)
::dpct::cs::memcpy(queue, nnz_ptr, &nnz_c_int, sizeof(int)).wait();
int row_ptr_c_0 =
(info_c->get_index_base() == oneapi::mkl::index_base::zero) ? 0 : 1;
nnz_c_int += row_ptr_c_0;
::dpct::cs::memcpy(queue, row_ptr_c + m, &nnz_c_int, sizeof(int));
::dpct::cs::memcpy(queue, row_ptr_c, &row_ptr_c_0, sizeof(int)).wait();
}
/// Computes the matrix C in the following operation:
/// C = alpha * A * B + beta * D
/// \param [in] desc The descriptor of this calculation.
/// \param [in] m The rows number of A, D and C.
/// \param [in] n The columns number of B, D and C.
/// \param [in] k The columns number of A and rows number of B.
/// \param [in] alpha Scaling factor.
/// \param [in] info_a Matrix info of the matrix A.
/// \param [in] nnz_a Non-zero elements number of matrix A.
/// \param [in] val_a An array containing the non-zero elements of the matrix A.
/// \param [in] row_ptr_a An array of length \p m + 1.
/// \param [in] col_ind_a An array containing the column indices in index-based
/// numbering.
/// \param [in] info_b Matrix info of the matrix B.
/// \param [in] nnz_b Non-zero elements number of matrix B.
/// \param [in] val_b An array containing the non-zero elements of the matrix B.
/// \param [in] row_ptr_b An array of length \p k + 1.
/// \param [in] col_ind_b An array containing the column indices in index-based
/// numbering.
/// \param [in] beta Scaling factor.
/// \param [in] info_d Matrix info of the matrix D.
/// \param [in] nnz_d Non-zero elements number of matrix D.
/// \param [in] val_d An array containing the non-zero elements of the matrix D.
/// \param [in] row_ptr_d An array of length \p m + 1.
/// \param [in] col_ind_d An array containing the column indices in index-based
/// numbering.
/// \param [in] info_c Matrix info of the matrix C.
/// \param [out] val_c An array containing the non-zero elements of the matrix
/// C.
/// \param [in] row_ptr_c An array of length \p m + 1.
/// \param [out] col_ind_c An array containing the column indices in index-based
/// numbering.
/// \param [in] info The information of csrgemm2 operation.
/// \param [in] buffer Workspace memory.
template <typename T>
void csrgemm2(descriptor_ptr desc, int m, int n, int k, const T *alpha,
const std::shared_ptr<matrix_info> info_a, int nnz_a,
const T *val_a, const int *row_ptr_a, const int *col_ind_a,
const std::shared_ptr<matrix_info> info_b, int nnz_b,
const T *val_b, const int *row_ptr_b, const int *col_ind_b,
const T *beta, const std::shared_ptr<matrix_info> info_d,
int nnz_d, const T *val_d, const int *row_ptr_d,
const int *col_ind_d, const std::shared_ptr<matrix_info> info_c,
T *val_c, const int *row_ptr_c, int *col_ind_c,
std::shared_ptr<csrgemm2_info> info, void *buffer) {
using Ty = typename ::dpct::detail::lib_data_traits_t<T>;
sycl::queue &queue = desc->get_queue();
auto alpha_value =
dpct::detail::get_value(reinterpret_cast<const Ty *>(alpha), queue);
auto beta_value =
dpct::detail::get_value(reinterpret_cast<const Ty *>(beta), queue);
info->matrix_handle_a.set_matrix_data<Ty>(m, k, info_a->get_index_base(),
row_ptr_a, col_ind_a, val_a);
info->matrix_handle_b.set_matrix_data<Ty>(k, n, info_b->get_index_base(),
row_ptr_b, col_ind_b, val_b);
__MATMAT(compute, nullptr, nullptr);
__MATMAT(finalize, nullptr, nullptr);
info->matrix_handle_d.set_matrix_data<Ty>(m, n, info_d->get_index_base(),
row_ptr_d, col_ind_d, val_d);
info->matrix_handle_c.set_matrix_data<Ty>(m, n, info_c->get_index_base(),
row_ptr_c, col_ind_c, val_c);
sycl::event e;
#ifndef DPCT_USM_LEVEL_NONE
e =
#endif
oneapi::mkl::sparse::omatadd(
queue, oneapi::mkl::transpose::nontrans,
oneapi::mkl::transpose::nontrans, alpha_value,
info->matrix_handle_axb.get_handle(), beta_value,
info->matrix_handle_d.get_handle(),
info->matrix_handle_c.get_handle(),
oneapi::mkl::sparse::omatadd_alg::default_alg,
info->omatadd_desc.get_handle());
std::vector<sycl::event> events;
events.push_back(e);
events.push_back(info->release(queue, e));
queue.submit([&](sycl::handler &cgh) {
cgh.depends_on(events);
cgh.host_task([_p1 = info->row_ptr_axb, _p2 = info->col_ind_axb,
_p3 = info->val_axb, _p4 = info->temp_buffer_1_size,
_p5 = info->temp_buffer_1, _p6 = info->temp_buffer_2_size,
_p7 = info->temp_buffer_2, _q = queue] {
::dpct::cs::free(_p1, _q);
::dpct::cs::free(_p2, _q);
::dpct::cs::free(_p3, _q);
#ifdef DPCT_USM_LEVEL_NONE
delete _p4;
delete _p5;
delete _p6;
delete _p7;
#else
sycl::free(_p4, _q);
sycl::free(_p5, _q);
sycl::free(_p6, _q);
sycl::free(_p7, _q);
#endif
});
});
}
#endif
} // namespace dpct::sparse
#endif // __DPCT_SPARSE_UTILS_HPP__