Skip to content

πŸ“„ Awesome OCR multiple programing languages toolkits based on ONNX Runtime, OpenVINO, MNN, PaddlePaddle, TensorRT and PyTorch.

onnxruntime header

rootc664cd0f4e10
diff --git a/include/onnxruntime/core/common/code_location.h b/include/onnxruntime/core/common/code_location.h
new file mode 100644
index 0000000..ff6506c
--- /dev/null
+++ b/include/onnxruntime/core/common/code_location.h
@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace onnxruntime {
+/**
+ CodeLocation captures information on where in the source code a message came from.
+*/
+struct CodeLocation {
+ /**
+ @param file_path Usually the value of __FILE__
+ @param line Usually the value of __LINE__
+ @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__
+ */
+ CodeLocation(const char* file_path, const int line, const char* func)
+ : file_and_path{file_path}, line_num{line}, function{func} {
+ }
+
+ /**
+ @param file_path Usually the value of __FILE__
+ @param line Usually the value of __LINE__
+ @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__
+ @param stacktrace Stacktrace from source of message.
+ */
+ CodeLocation(const char* file_path, const int line, const char* func, const std::vector<std::string>& stacktrace)
+ : file_and_path{file_path}, line_num{line}, function{func}, stacktrace(stacktrace) {
+ }
+
+ std::string FileNoPath() const {
+ // assuming we always have work to do, so not trying to avoid creating a new string if
+ // no path was removed.
+ return file_and_path.substr(file_and_path.find_last_of("/\\") + 1);
+ }
+
+ enum Format {
+ kFilename,
+ kFilenameAndPath
+ };
+
+ std::string ToString(Format format = Format::kFilename) const {
+ std::ostringstream out;
+ out << (format == Format::kFilename ? FileNoPath() : file_and_path) << ":" << line_num << " " << function;
+ return out.str();
+ }
+
+ const std::string file_and_path;
+ const int line_num;
+ const std::string function;
+ const std::vector<std::string> stacktrace;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h
new file mode 100644
index 0000000..2d9dc34
--- /dev/null
+++ b/include/onnxruntime/core/common/common.h
@@ -0,0 +1,289 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// Portions Copyright (c) Microsoft Corporation
+
+#pragma once
+
+#include <algorithm>
+#include <cstring>
+#include <climits>
+#include <functional>
+#include <memory>
+#include <numeric>
+#include <set>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+#include <chrono>
+
+#include "core/common/code_location.h"
+#include "core/common/exceptions.h"
+#include "core/common/make_string.h"
+#include "core/common/make_unique.h"
+#include "core/common/status.h"
+
+#ifdef USE_MIMALLOC_ARENA_ALLOCATOR
+#include <mimalloc.h>
+#endif
+
+#ifdef ORT_NO_EXCEPTIONS
+#include <iostream>
+#endif
+
+namespace onnxruntime {
+
+using TimePoint = std::chrono::high_resolution_clock::time_point;
+
+// Using statements for common classes that we refer to in ONNXRuntime very often.
+// TODO(Task:137) Remove 'using' statements from header files
+using common::Status;
+
+#ifdef _WIN32
+#define ORT_UNUSED_PARAMETER(x) (x)
+#else
+#define ORT_UNUSED_PARAMETER(x) (void)(x)
+#endif
+
+#ifndef ORT_HAVE_ATTRIBUTE
+#ifdef __has_attribute
+#define ORT_HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+#define ORT_HAVE_ATTRIBUTE(x) 0
+#endif
+#endif
+
+// ORT_ATTRIBUTE_UNUSED
+//
+// Prevents the compiler from complaining about or optimizing away variables
+// that appear unused on Linux
+#if ORT_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__))
+#undef ORT_ATTRIBUTE_UNUSED
+#define ORT_ATTRIBUTE_UNUSED __attribute__((__unused__))
+#else
+#define ORT_ATTRIBUTE_UNUSED
+#endif
+
+// macro to explicitly ignore the return value from a function call so Code Analysis doesn't complain
+#define ORT_IGNORE_RETURN_VALUE(fn) \
+ static_cast<void>(fn)
+
+std::vector<std::string> GetStackTrace();
+// these is a helper function that gets defined by platform/Telemetry
+void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,
+ const char* function, uint32_t line);
+
+// __PRETTY_FUNCTION__ isn't a macro on gcc, so use a check for _MSC_VER
+// so we only define it as one for MSVC
+#if (_MSC_VER && !defined(__PRETTY_FUNCTION__))
+#define __PRETTY_FUNCTION__ __FUNCTION__
+#endif
+
+// Capture where a message is coming from. Use __FUNCTION__ rather than the much longer __PRETTY_FUNCTION__
+#define ORT_WHERE \
+ ::onnxruntime::CodeLocation(__FILE__, __LINE__, __FUNCTION__)
+
+#define ORT_WHERE_WITH_STACK \
+ ::onnxruntime::CodeLocation(__FILE__, __LINE__, __PRETTY_FUNCTION__, ::onnxruntime::GetStackTrace())
+
+#ifdef ORT_NO_EXCEPTIONS
+
+#define ORT_TRY if (true)
+#define ORT_CATCH(x) else if (false)
+#define ORT_RETHROW
+
+// In order to ignore the catch statement when a specific exception (not ... ) is caught and referred
+// in the body of the catch statements, it is necessary to wrap the body of the catch statement into
+// a lambda function. otherwise the exception referred will be undefined and cause build break
+#define ORT_HANDLE_EXCEPTION(func)
+
+// TODO, consider changing the output of the error message from std::cerr to logging when the
+// exceptions are disabled, since using std::cerr might increase binary size, and std::cerr output
+// might not be easily accesible on some systems such as mobile
+
+// Throw an exception with optional message.
+// NOTE: The arguments get streamed into a string via ostringstream::operator<<
+// DO NOT use a printf format string, as that will not work as you expect.
+#define ORT_THROW(...) \
+ do { \
+ std::cerr << ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, \
+ ::onnxruntime::MakeString(__VA_ARGS__)) \
+ .what() \
+ << std::endl; \
+ abort(); \
+ } while (false)
+
+// Just in order to mark things as not implemented. Do not use in final code.
+#define ORT_NOT_IMPLEMENTED(...) \
+ do { \
+ std::cerr << ::onnxruntime::NotImplementedException(::onnxruntime::MakeString(__VA_ARGS__)) \
+ .what() \
+ << std::endl; \
+ abort(); \
+ } while (false)
+
+// Check condition.
+// NOTE: The arguments get streamed into a string via ostringstream::operator<<
+// DO NOT use a printf format string, as that will not work as you expect.
+#define ORT_ENFORCE(condition, ...) \
+ do { \
+ if (!(condition)) { \
+ std::cerr << ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, #condition, \
+ ::onnxruntime::MakeString(__VA_ARGS__)) \
+ .what() \
+ << std::endl; \
+ abort(); \
+ } \
+ } while (false)
+
+#define ORT_THROW_EX(ex, ...) \
+ do { \
+ std::cerr << #ex << "(" << ::onnxruntime::MakeString(__VA_ARGS__) << ")" << std::endl; \
+ abort(); \
+ } while (false)
+
+#else
+
+#define ORT_TRY try
+#define ORT_CATCH(x) catch (x)
+#define ORT_RETHROW throw;
+
+#define ORT_HANDLE_EXCEPTION(func) func()
+
+// Throw an exception with optional message.
+// NOTE: The arguments get streamed into a string via ostringstream::operator<<
+// DO NOT use a printf format string, as that will not work as you expect.
+#define ORT_THROW(...) \
+ throw ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, ::onnxruntime::MakeString(__VA_ARGS__))
+
+// Just in order to mark things as not implemented. Do not use in final code.
+#define ORT_NOT_IMPLEMENTED(...) \
+ throw ::onnxruntime::NotImplementedException(::onnxruntime::MakeString(__VA_ARGS__))
+
+// Check condition.
+// NOTE: The arguments get streamed into a string via ostringstream::operator<<
+// DO NOT use a printf format string, as that will not work as you expect.
+#define ORT_ENFORCE(condition, ...) \
+ if (!(condition)) \
+ throw ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, #condition, \
+ ::onnxruntime::MakeString(__VA_ARGS__))
+
+#define ORT_THROW_EX(ex, ...) \
+ throw ex(__VA_ARGS__)
+
+#endif
+
+#define ORT_MAKE_STATUS(category, code, ...) \
+ ::onnxruntime::common::Status(::onnxruntime::common::category, \
+ ::onnxruntime::common::code, \
+ ::onnxruntime::MakeString(__VA_ARGS__))
+
+// Check condition. if met, return status.
+#define ORT_RETURN_IF(condition, ...) \
+ if (condition) { \
+ return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, \
+ "Satisfied, but should not be: " #condition "\n", \
+ ORT_WHERE.ToString(), ::onnxruntime::MakeString(__VA_ARGS__)); \
+ }
+
+// Check condition. if not met, return status.
+#define ORT_RETURN_IF_NOT(condition, ...) \
+ if (!(condition)) { \
+ return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Not satisfied: " #condition "\n", \
+ ORT_WHERE.ToString(), ::onnxruntime::MakeString(__VA_ARGS__)); \
+ }
+
+// Macros to disable the copy and/or move ctor and assignment methods
+// These are usually placed in the private: declarations for a class.
+
+#define ORT_DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete
+
+#define ORT_DISALLOW_ASSIGNMENT(TypeName) TypeName& operator=(const TypeName&) = delete
+
+#define ORT_DISALLOW_COPY_AND_ASSIGNMENT(TypeName) \
+ ORT_DISALLOW_COPY(TypeName); \
+ ORT_DISALLOW_ASSIGNMENT(TypeName)
+
+#define ORT_DISALLOW_MOVE(TypeName) \
+ TypeName(TypeName&&) = delete; \
+ TypeName& operator=(TypeName&&) = delete
+
+#define ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TypeName) \
+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(TypeName); \
+ ORT_DISALLOW_MOVE(TypeName)
+
+#define ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id) \
+ do { \
+ auto _status = (expr); \
+ if ((!_status.IsOK())) { \
+ ::onnxruntime::LogRuntimeError(session_id, _status, __FILE__, __FUNCTION__, __LINE__); \
+ return _status; \
+ } \
+ } while (0)
+
+#define ORT_RETURN_IF_ERROR_SESSIONID_(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id_)
+#define ORT_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, 0)
+
+#define ORT_THROW_IF_ERROR(expr) \
+ do { \
+ auto _status = (expr); \
+ if ((!_status.IsOK())) { \
+ ::onnxruntime::LogRuntimeError(0, _status, __FILE__, __FUNCTION__, __LINE__); \
+ ORT_THROW(_status); \
+ } \
+ } while (0)
+
+// use this macro when cannot early return
+#define ORT_CHECK_AND_SET_RETVAL(expr) \
+ do { \
+ if (retval.IsOK()) { \
+ retval = (expr); \
+ } \
+ } while (0)
+
+// C++ Core Guideline check suppression.
+#if defined(_MSC_VER) && !defined(__NVCC__)
+#define GSL_SUPPRESS(tag) [[gsl::suppress(tag)]]
+#else
+#define GSL_SUPPRESS(tag)
+#endif
+
+inline long long TimeDiffMicroSeconds(TimePoint start_time) {
+ auto end_time = std::chrono::high_resolution_clock::now();
+ return std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
+}
+
+inline long long TimeDiffMicroSeconds(TimePoint start_time, TimePoint end_time) {
+ return std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();
+}
+
+struct null_type {};
+inline std::string ToMBString(const std::string& s) { return s; }
+#ifdef _WIN32
+/**
+ * Convert a wide character string into a narrow one, with local ANSI code page(like CP936)
+ * DO NOT assume the result string is encoded in UTF-8
+ */
+std::string ToMBString(const std::wstring& s);
+
+std::wstring ToWideString(const std::string& s);
+inline std::wstring ToWideString(const std::wstring& s) { return s; }
+#else
+inline std::string ToWideString(const std::string& s) { return s; }
+#endif
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/const_pointer_container.h b/include/onnxruntime/core/common/const_pointer_container.h
new file mode 100644
index 0000000..1d821ba
--- /dev/null
+++ b/include/onnxruntime/core/common/const_pointer_container.h
@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <type_traits>
+
+namespace onnxruntime {
+/**
+ Container has T* entries. e.g. std::vector<T*>, and this class provides const access to those
+ via iterators and direct access, as the standard behavior only makes the pointer constant,
+ and not what is pointed too. i.e. you get a const pointer to T not a pointer to const T without this wrapper.
+ See https://stackoverflow.com/questions/8017036/understanding-const-iterator-with-pointers
+*/
+template <typename Container>
+class ConstPointerContainer {
+ public:
+ using T = typename std::remove_pointer<typename Container::value_type>::type;
+
+ class ConstIterator {
+ public:
+ using const_iterator = typename Container::const_iterator;
+ using iterator_category = std::input_iterator_tag;
+ using value_type = T*;
+ using difference_type = std::ptrdiff_t;
+ using pointer = T**;
+ using reference = T*&;
+
+ /** Construct iterator for container that will return const T* entries.*/
+ explicit ConstIterator(const_iterator position) noexcept : current_{position}, item_{nullptr} {}
+ ConstIterator(const ConstIterator& other) = default;
+ ConstIterator& operator=(const ConstIterator& other) = default;
+
+ bool operator==(const ConstIterator& other) const noexcept { return current_ == other.current_; }
+ bool operator!=(const ConstIterator& other) const noexcept { return current_ != other.current_; }
+
+ ConstIterator& operator++() {
+ ++current_;
+ return *this;
+ }
+
+ ConstIterator operator++(int) {
+ ConstIterator tmp{*this};
+ ++(*this);
+ return tmp;
+ }
+
+ const T*& operator*() const {
+ item_ = *current_;
+ return item_;
+ }
+
+ const T** operator->() const { return &(operator*()); };
+
+ private:
+ const_iterator current_;
+ mutable const T* item_;
+ };
+
+ /**
+ Construct wrapper class that will provide const access to the pointers in a container of non-const pointers.
+ @param data Container with non-const pointers. e.g. std::vector<T*>
+ */
+ explicit ConstPointerContainer(const Container& data) noexcept : data_(data) {}
+
+ size_t size() const noexcept { return data_.size(); }
+ bool empty() const noexcept { return data_.empty(); }
+
+ ConstIterator cbegin() const noexcept { return ConstIterator(data_.cbegin()); }
+ ConstIterator cend() const noexcept { return ConstIterator(data_.cend()); }
+
+ ConstIterator begin() const noexcept { return ConstIterator(data_.cbegin()); }
+ ConstIterator end() const noexcept { return ConstIterator(data_.cend()); }
+
+ const T* operator[](size_t index) const { return data_[index]; }
+
+ const T* at(size_t index) const {
+ ORT_ENFORCE(index < data_.size());
+ return data_[index];
+ }
+
+ private:
+ const Container& data_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/denormal.h b/include/onnxruntime/core/common/denormal.h
new file mode 100644
index 0000000..baac109
--- /dev/null
+++ b/include/onnxruntime/core/common/denormal.h
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+
+// Set or unset flush-to-zero and denormal=as-zero if SSE3 instructions are supported.
+// Return true if SSE3 instruction is supported, otherwise return false.
+bool SetDenormalAsZero(bool on);
+
+#ifdef _OPENMP
+// Set flush-to-zero and denormal-as-zero on OpenMP threads when on is true.
+void InitializeWithDenormalAsZero(bool on);
+#endif
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/eigen_common_wrapper.h b/include/onnxruntime/core/common/eigen_common_wrapper.h
new file mode 100644
index 0000000..fa15399
--- /dev/null
+++ b/include/onnxruntime/core/common/eigen_common_wrapper.h
@@ -0,0 +1,41 @@
+//-----------------------------------------------------------------------------
+//
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+//
+//-----------------------------------------------------------------------------
+#pragma once
+#include "onnxruntime_config.h"
+// build/external/eigen/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h:162:71:
+// error: ignoring attributes on template argument "Eigen::PacketType<const float, Eigen::DefaultDevice>::type {aka __vector(4) float}" [-Werror=ignored-attributes]
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#if __GNUC__ >= 6
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#ifdef HAS_DEPRECATED_COPY
+#pragma GCC diagnostic ignored "-Wdeprecated-copy"
+#endif
+#elif defined(_MSC_VER)
+// build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76):
+// warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence
+
+// unsupported\eigen\cxx11\src\Tensor\TensorUInt128.h(150,0): Warning C4245: 'initializing': conversion from '__int64'
+// to 'uint64_t', signed/unsigned mismatch
+#pragma warning(push)
+#pragma warning(disable : 4554)
+#pragma warning(disable : 4245)
+#pragma warning(disable : 4127)
+#pragma warning(disable : 4805)
+#pragma warning(disable : 6313)
+#pragma warning(disable : 6294)
+#endif
+
+#include "unsupported/Eigen/CXX11/Tensor"
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
diff --git a/include/onnxruntime/core/common/exceptions.h b/include/onnxruntime/core/common/exceptions.h
new file mode 100644
index 0000000..cbebc88
--- /dev/null
+++ b/include/onnxruntime/core/common/exceptions.h
@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <algorithm>
+#include <exception>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "core/common/common.h"
+#include "core/common/code_location.h"
+
+namespace onnxruntime {
+
+class NotImplementedException : public std::logic_error {
+ public:
+ explicit NotImplementedException(const char* _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){};
+ explicit NotImplementedException(const std::string& _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){};
+};
+
+class TypeMismatchException : public std::logic_error {
+ public:
+ TypeMismatchException() noexcept : logic_error("Type mismatch"){};
+};
+
+class OnnxRuntimeException : public std::exception {
+ public:
+ OnnxRuntimeException(const CodeLocation& location, const std::string& msg) noexcept
+ : OnnxRuntimeException(location, nullptr, msg) {
+ }
+
+ /**
+ Create a new exception that captures the location it was thrown from.
+ @param location Location in the source code the exception is being thrown from
+ @param failed_condition Optional string containing the condition that failed.
+ e.g. "tensor.Size() == input.Size()". May be nullptr.
+ @param msg Message containing additional information about the exception cause.
+ */
+ OnnxRuntimeException(const CodeLocation& location, const char* failed_condition, const std::string& msg)
+ : location_{location} {
+ std::ostringstream ss;
+
+ ss << location.ToString(CodeLocation::kFilenameAndPath); // output full path in case just the filename is ambiguous
+ if (failed_condition != nullptr) {
+ ss << " " << failed_condition << " was false.";
+ }
+
+ ss << " " << msg << "\n";
+ if (!location.stacktrace.empty()) {
+ ss << "Stacktrace:\n";
+ // skip the first entry in the stacktrace as we have that information from location.ToString()
+ std::copy(++location.stacktrace.begin(), location.stacktrace.end(), std::ostream_iterator<std::string>(ss, "\n"));
+ }
+
+ what_ = ss.str();
+ }
+
+ const char* what() const noexcept override {
+ return what_.c_str();
+ }
+
+ private:
+ const CodeLocation location_;
+ const std::vector<std::string> stacktrace_;
+ std::string what_;
+};
+
+} // namespace onnxruntime
\ No newline at end of file
diff --git a/include/onnxruntime/core/common/logging/capture.h b/include/onnxruntime/core/common/logging/capture.h
new file mode 100644
index 0000000..4f71bb3
--- /dev/null
+++ b/include/onnxruntime/core/common/logging/capture.h
@@ -0,0 +1,115 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <cstdarg>
+#include <gsl/gsl>
+#include "core/common/common.h"
+#include "core/common/code_location.h"
+#include "core/common/logging/severity.h"
+
+namespace onnxruntime {
+namespace logging {
+
+class Logger;
+enum class DataType;
+
+/**
+ Class to capture the details of a log message.
+*/
+class Capture {
+ public:
+ /**
+ Initializes a new instance of the Capture class.
+ @param logger The logger.
+ @param severity The severity.
+ @param category The category.
+ @param dataType Type of the data.
+ @param location The file location the log message is coming from.
+ */
+ Capture(const Logger& logger, logging::Severity severity, const char* category,
+ logging::DataType dataType, const CodeLocation& location)
+ : logger_{&logger}, severity_{severity}, category_{category}, data_type_{dataType}, location_{location} {
+ }
+
+ /**
+ The stream that can capture the message via operator<<.
+ @returns Output stream.
+ */
+ std::ostream& Stream() noexcept {
+ return stream_;
+ }
+
+#ifdef _MSC_VER
+// add SAL annotation for printf format string. requires Code Analysis to run to validate usage.
+#define msvc_printf_check _Printf_format_string_
+#define __attribute__(x) // Disable for MSVC. Supported by GCC and CLang.
+#else
+#define msvc_printf_check
+#endif
+
+ /**
+ Captures a printf style log message.
+ @param name="format">The printf format.
+ @param name="">Arguments to the printf format if needed.
+ @remarks
+ A maximum of 2K of output will be captured currently.
+ Non-static method, so 'this' is implicit first arg, and we use format(printf(2,3)
+ */
+ void CapturePrintf(msvc_printf_check const char* format, ...) __attribute__((format(printf, 2, 3)));
+
+ /**
+ Process a printf style log message.
+ @param format The printf format.
+ @param ... Arguments to the printf format if needed.
+ @remarks
+ A maximum of 2K of output will be captured currently.
+ Note: As va_list is 'char *', we have to disambiguate this from CapturePrintf
+ so that something like "One string: %s", "the string" does not consider "the string"
+ to be the va_list.
+ */
+ void ProcessPrintf(msvc_printf_check const char* format, va_list args);
+
+ logging::Severity Severity() const noexcept {
+ return severity_;
+ }
+
+ char SeverityPrefix() const noexcept {
+ // Carefully setup so severity_ is a valid index
+ GSL_SUPPRESS(bounds .2) {
+ return logging::SEVERITY_PREFIX[static_cast<int>(severity_)];
+ }
+ }
+
+ const char* Category() const noexcept {
+ return category_;
+ }
+
+ logging::DataType DataType() const noexcept {
+ return data_type_;
+ }
+
+ const CodeLocation& Location() const noexcept {
+ return location_;
+ }
+
+ std::string Message() const noexcept {
+ return stream_.str();
+ }
+
+ ~Capture();
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Capture);
+
+ const Logger* logger_;
+ const logging::Severity severity_;
+ const char* category_;
+ const logging::DataType data_type_;
+ const CodeLocation location_;
+
+ std::ostringstream stream_;
+};
+} // namespace logging
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/logging/isink.h b/include/onnxruntime/core/common/logging/isink.h
new file mode 100644
index 0000000..a67777d
--- /dev/null
+++ b/include/onnxruntime/core/common/logging/isink.h
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+
+#include "core/common/logging/logging.h"
+
+namespace onnxruntime {
+namespace logging {
+class ISink {
+ public:
+ ISink() = default;
+
+ /**
+ Sends the message to the sink.
+ @param timestamp The timestamp.
+ @param logger_id The logger identifier.
+ @param message The captured message.
+ */
+ void Send(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) {
+ SendImpl(timestamp, logger_id, message);
+ }
+
+ /**
+ Sends a Profiling Event Record to the sink.
+ @param Profiling Event Record
+ */
+ virtual void SendProfileEvent(profiling::EventRecord&) const {};
+
+ virtual ~ISink() = default;
+
+ private:
+ // Make Code Analysis happy by disabling all for now. Enable as needed.
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ISink);
+
+ virtual void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) = 0;
+};
+} // namespace logging
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/logging/logging.h b/include/onnxruntime/core/common/logging/logging.h
new file mode 100644
index 0000000..96c07f0
--- /dev/null
+++ b/include/onnxruntime/core/common/logging/logging.h
@@ -0,0 +1,347 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+
+#include "core/common/common.h"
+#include "core/common/logging/capture.h"
+#include "core/common/logging/severity.h"
+
+#include "core/common/logging/macros.h"
+
+/*
+
+ Logging overview and expected usage:
+
+ At program startup:
+ * Create one or more ISink instances. If multiple, combine using composite_sink.
+ * Create a LoggingManager instance with the sink/s with is_default_instance set to true
+ * Only one instance should be created in this way, and it should remain valid for
+ until the program no longer needs to produce log output.
+
+ You can either use the static default Logger which LoggingManager will create when constructed
+ via LoggingManager::DefaultLogger(), or separate Logger instances each with different log ids
+ via LoggingManager::CreateLogger.
+
+ The log id is passed to the ISink instance with the sink determining how the log id is used
+ in the output.
+
+ LoggingManager
+ * creates the Logger instances used by the application
+ * provides a static default logger instance
+ * owns the log sink instance
+ * applies checks on severity and output of user data
+
+ The log macros create a Capture instance to capture the information to log.
+ If the severity and/or user filtering settings would prevent logging, no evaluation
+ of the log arguments will occur, so no performance cost beyond the severity and user
+ filtering check.
+
+ A sink can do further filter as needed.
+
+*/
+
+namespace onnxruntime {
+namespace profiling {
+
+enum EventCategory {
+ SESSION_EVENT = 0,
+ NODE_EVENT,
+ EVENT_CATEGORY_MAX
+};
+
+/*
+Event descriptions for the above session events.
+*/
+static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = {
+ "Session",
+ "Node"};
+
+/*
+Timing record for all events.
+*/
+struct EventRecord {
+ EventRecord(EventCategory category,
+ int process_id,
+ int thread_id,
+ std::string event_name,
+ long long time_stamp,
+ long long duration,
+ std::unordered_map<std::string, std::string>&& event_args) : cat(category),
+ pid(process_id),
+ tid(thread_id),
+ name(std::move(event_name)),
+ ts(time_stamp),
+ dur(duration),
+ args(event_args) {}
+ EventCategory cat;
+ int pid;
+ int tid;
+ std::string name;
+ long long ts;
+ long long dur;
+ std::unordered_map<std::string, std::string> args;
+};
+} // namespace profiling
+
+namespace logging {
+
+using Timestamp = std::chrono::time_point<std::chrono::system_clock>;
+
+#ifndef NDEBUG
+ORT_ATTRIBUTE_UNUSED static bool vlog_enabled = true; // Set directly based on your needs.
+#else
+constexpr bool vlog_enabled = false; // no VLOG output
+#endif
+
+enum class DataType {
+ SYSTEM = 0, ///< System data.
+ USER = 1 ///< Contains potentially sensitive user data.
+};
+
+// Internal log categories.
+// Logging interface takes const char* so arbitrary values can also be used.
+struct Category {
+ static const char* onnxruntime; ///< General output
+ static const char* System; ///< Log output regarding interactions with the host system
+ // TODO: What other high level categories are meaningful? Model? Optimizer? Execution?
+};
+
+class ISink;
+class Logger;
+class Capture;
+
+/// <summary>
+/// The logging manager.
+/// Owns the log sink and potentially provides a default Logger instance.
+/// Provides filtering based on a minimum LogSeverity level, and of messages with DataType::User if enabled.
+/// </summary>
+class LoggingManager final {
+ public:
+ enum InstanceType {
+ Default, ///< Default instance of LoggingManager that should exist for the lifetime of the program
+ Temporal ///< Temporal instance. CreateLogger(...) should be used, however DefaultLogger() will NOT be provided via this instance.
+ };
+
+ /**
+ Initializes a new instance of the LoggingManager class.
+ @param sink The sink to write to. Use CompositeSink if you need to write to multiple places.
+ @param default_min_severity The default minimum severity. Messages with lower severity will be ignored unless
+ overridden in CreateLogger.
+ @param default_filter_user_data If set to true ignore messages with DataType::USER unless overridden in CreateLogger.
+ @param instance_type If InstanceType::Default, this is the default instance of the LoggingManager
+ and is expected to exist for the lifetime of the program.
+ It creates and owns the default logger that calls to the static DefaultLogger method return.
+ @param default_logger_id Logger Id to use for the default logger. nullptr/ignored if instance_type == Temporal.
+ @param default_max_vlog_level Default maximum level for VLOG messages to be created unless overridden in CreateLogger.
+ Requires a severity of kVERBOSE for VLOG messages to be logged.
+ */
+ LoggingManager(std::unique_ptr<ISink> sink, Severity default_min_severity, bool default_filter_user_data,
+ InstanceType instance_type,
+ const std::string* default_logger_id = nullptr,
+ int default_max_vlog_level = -1);
+
+ /**
+ Creates a new logger instance which will use the provided logger_id and default severity and vlog levels.
+ @param logger_id The log identifier.
+ @returns A new Logger instance that the caller owns.
+ */
+ std::unique_ptr<Logger> CreateLogger(const std::string& logger_id);
+
+ /**
+ Creates a new logger instance which will use the provided logger_id, severity and vlog levels.
+ @param logger_id The log identifier.
+ @param min_severity The minimum severity. Requests to create messages with lower severity will be ignored.
+ @param filter_user_data If set to true ignore messages with DataType::USER.
+ @param max_vlog_level Maximum level for VLOG messages to be created.
+ @returns A new Logger instance that the caller owns.
+ */
+ std::unique_ptr<Logger> CreateLogger(const std::string& logger_id,
+ Severity min_severity, bool filter_user_data, int max_vlog_level = -1);
+
+ /**
+ Gets the default logger instance if set. Throws if no default logger is currently registered.
+ @remarks
+ Creating a LoggingManager instance with is_default_instance == true registers a default logger.
+ Note that the default logger is only valid until the LoggerManager that registered it is destroyed.
+ @returns The default logger if available.
+ */
+ static const Logger& DefaultLogger();
+
+ /**
+ Change the minimum severity level for log messages to be output by the default logger.
+ @param severity The severity.
+ */
+ static void SetDefaultLoggerSeverity(Severity severity);
+
+ /**
+ Logs a FATAL level message and creates an exception that can be thrown with error information.
+ @param category The log category.
+ @param location The location the log message was generated.
+ @param format_str The printf format string.
+ @param ... The printf arguments.
+ @returns A new Logger instance that the caller owns.
+ */
+ static std::exception LogFatalAndCreateException(const char* category,
+ const CodeLocation& location,
+ const char* format_str, ...);
+
+ /**
+ Logs the message using the provided logger id.
+ @param logger_id The log identifier.
+ @param message The log message.
+ */
+ void Log(const std::string& logger_id, const Capture& message) const;
+
+ /**
+ Sends a Profiling Event Record to the sink.
+ @param Profiling Event Record
+ */
+ void SendProfileEvent(profiling::EventRecord& eventRecord) const;
+ ~LoggingManager();
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(LoggingManager);
+
+ Timestamp GetTimestamp() const noexcept;
+ void CreateDefaultLogger(const std::string& logger_id);
+
+ std::unique_ptr<ISink> sink_;
+ const Severity default_min_severity_;
+ const bool default_filter_user_data_;
+ const int default_max_vlog_level_;
+ bool owns_default_logger_;
+
+ static Logger* s_default_logger_;
+
+ struct Epochs {
+ const std::chrono::time_point<std::chrono::high_resolution_clock> high_res;
+ const std::chrono::time_point<std::chrono::system_clock> system;
+ const std::chrono::minutes localtime_offset_from_utc;
+ };
+
+ static const Epochs& GetEpochs() noexcept;
+};
+
+/**
+ Logger provides a per-instance log id. Everything else is passed back up to the LoggingManager
+*/
+class Logger {
+ public:
+ /**
+ Initializes a new instance of the Logger class.
+ @param loggingManager The logging manager.
+ @param id The identifier for messages coming from this Logger.
+ @param severity Minimum severity for messages to be created and logged.
+ @param filter_user_data Should USER data be filtered from output.
+ @param vlog_level Minimum level for VLOG messages to be created. Note that a severity of kVERBOSE must be provided
+ for VLOG messages to be logged.
+ */
+ Logger(const LoggingManager& loggingManager, std::string id,
+ Severity severity, bool filter_user_data, int vlog_level)
+ : logging_manager_{&loggingManager},
+ id_{id},
+ min_severity_{severity},
+ filter_user_data_{filter_user_data},
+ max_vlog_level_{severity > Severity::kVERBOSE ? -1 : vlog_level} { // disable unless logging VLOG messages
+ }
+
+ /**
+ Get the minimum severity level for log messages to be output.
+ @returns The severity.
+ */
+ Severity GetSeverity() const noexcept { return min_severity_; }
+
+ /**
+ Change the minimum severity level for log messages to be output.
+ @param severity The severity.
+ */
+ void SetSeverity(Severity severity) noexcept { min_severity_ = severity; }
+
+ /**
+ Check if output is enabled for the provided LogSeverity and DataType values.
+ @param severity The severity.
+ @param data_type Type of the data.
+ @returns True if a message with these values will be logged.
+ */
+ bool OutputIsEnabled(Severity severity, DataType data_type) const noexcept {
+ return (severity >= min_severity_ && (data_type != DataType::USER || !filter_user_data_));
+ }
+
+ /**
+ Return the maximum VLOG level allowed.
+ */
+ int VLOGMaxLevel() const noexcept {
+ return max_vlog_level_;
+ }
+
+ /**
+ Logs the captured message.
+ @param message The log message.
+ */
+ void Log(const Capture& message) const {
+ logging_manager_->Log(id_, message);
+ }
+
+ /**
+ Sends a Profiling Event Record to the sink.
+ @param Profiling Event Record
+ */
+ void SendProfileEvent(profiling::EventRecord& eventRecord) const {
+ logging_manager_->SendProfileEvent(eventRecord);
+ }
+
+ private:
+ const LoggingManager* logging_manager_;
+ const std::string id_;
+ Severity min_severity_;
+ const bool filter_user_data_;
+ const int max_vlog_level_;
+};
+
+inline const Logger& LoggingManager::DefaultLogger() {
+ if (s_default_logger_ == nullptr) {
+ // fail early for attempted misuse. don't use logging macros as we have no logger.
+ ORT_THROW("Attempt to use DefaultLogger but none has been registered.");
+ }
+
+ return *s_default_logger_;
+}
+
+inline void LoggingManager::SetDefaultLoggerSeverity(Severity severity) {
+ if (s_default_logger_ == nullptr) {
+ // fail early for attempted misuse. don't use logging macros as we have no logger.
+ ORT_THROW("Attempt to use DefaultLogger but none has been registered.");
+ }
+
+ s_default_logger_->SetSeverity(severity);
+}
+
+inline Timestamp LoggingManager::GetTimestamp() const noexcept {
+ static const Epochs& epochs = GetEpochs();
+
+ const auto high_res_now = std::chrono::high_resolution_clock::now();
+ return std::chrono::time_point_cast<std::chrono::system_clock::duration>(
+ epochs.system + (high_res_now - epochs.high_res) + epochs.localtime_offset_from_utc);
+}
+
+/**
+ Return the current thread id.
+*/
+unsigned int GetThreadId();
+
+/**
+ Return the current process id.
+*/
+unsigned int GetProcessId();
+
+} // namespace logging
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/logging/macros.h b/include/onnxruntime/core/common/logging/macros.h
new file mode 100644
index 0000000..570bc14
--- /dev/null
+++ b/include/onnxruntime/core/common/logging/macros.h
@@ -0,0 +1,209 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+// NOTE: Don't include this file directly. Include logging.h
+
+#define CREATE_MESSAGE(logger, severity, category, datatype) \
+ ::onnxruntime::logging::Capture(logger, ::onnxruntime::logging::Severity::k##severity, category, datatype, ORT_WHERE)
+
+/*
+ Both printf and stream style logging are supported.
+ Not that printf currently has a 2K limit to the message size.
+
+ LOGS_* macros are for stream style
+ LOGF_* macros are for printf style
+
+ The Message class captures the log input, and pushes it through the logger in its destructor.
+
+ Use the *FATAL* macros if you want a Severity::kFatal message to also throw.
+
+ There are a few variants to minimize the length of the macro name required in the calling code.
+ They are optimized so the shortest names are for the (expected) most common usage. This can be
+ tweaked if needed.
+
+ Explicit logger vs LoggingManager::DefaulLogger()
+ Default is for a logger instance to be explicitly passed in.
+ The logger instance provides an identifier so that log messages from different runs can be separated.
+
+ Variants with DEFAULT in the macro name use the default logger provided by logging manager. This is
+ static so accessible from any code, provided a LoggingManager instance created with InstanceType::Default
+ exists somewhere. See logging.h for further explanation of the expected setup.
+
+ DataType
+ Default uses DataType::SYSTEM.
+
+ Variants with USER in the macro name use DataType::USER. This is data that could be PII, and may need to
+ be filtered from output. LoggingManager applies this filtering.
+
+ Category
+ Default category is ::onnxruntime::Logging::Category::onnxruntime.
+
+ If you wish to provide a different category, use variants with CATEGORY in the macro name
+
+*/
+
+// Logging with explicit category
+
+// iostream style logging. Capture log info in Message, and push to the logger in ~Message.
+#define LOGS_CATEGORY(logger, severity, category) \
+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \
+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).Stream()
+
+#define LOGS_USER_CATEGORY(logger, severity, category) \
+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \
+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).Stream()
+
+ // printf style logging. Capture log info in Message, and push to the logger in ~Message.
+#define LOGF_CATEGORY(logger, severity, category, format_str, ...) \
+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \
+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).CapturePrintf(format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_CATEGORY(logger, severity, category, format_str, ...) \
+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \
+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).CapturePrintf(format_str, ##__VA_ARGS__)
+
+ // Logging with category of "onnxruntime"
+
+#define LOGS(logger, severity) \
+ LOGS_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGS_USER(logger, severity) \
+ LOGS_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+ // printf style logging. Capture log info in Message, and push to the logger in ~Message.
+#define LOGF(logger, severity, format_str, ...) \
+ LOGF_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER(logger, severity, format_str, ...) \
+ LOGF_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+ /*
+
+ Macros that use the default logger.
+ A LoggingManager instance must be currently valid for the default logger to be available.
+
+ */
+
+ // Logging with explicit category
+
+#define LOGS_DEFAULT_CATEGORY(severity, category) \
+ LOGS_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category)
+
+#define LOGS_USER_DEFAULT_CATEGORY(severity, category) \
+ LOGS_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category)
+
+#define LOGF_DEFAULT_CATEGORY(severity, category, format_str, ...) \
+ LOGF_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ...) \
+ LOGF_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__)
+
+// Logging with category of "onnxruntime"
+
+#define LOGS_DEFAULT(severity) \
+ LOGS_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGS_USER_DEFAULT(severity) \
+ LOGS_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGF_DEFAULT(severity, format_str, ...) \
+ LOGF_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_DEFAULT(severity, format_str, ...) \
+ LOGF_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+ /*
+
+ Conditional logging
+
+ */
+
+ // Logging with explicit category
+
+#define LOGS_CATEGORY_IF(boolean_expression, logger, severity, category) \
+ if ((boolean_expression) == true) LOGS_CATEGORY(logger, severity, category)
+
+#define LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \
+ if ((boolean_expression) == true) LOGS_DEFAULT_CATEGORY(severity, category)
+
+#define LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, category) \
+ if ((boolean_expression) == true) LOGS_USER_CATEGORY(logger, severity, category)
+
+#define LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \
+ if ((boolean_expression) == true) LOGS_USER_DEFAULT_CATEGORY(severity, category)
+
+#define LOGF_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \
+ if ((boolean_expression) == true) LOGF_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__)
+
+#define LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \
+ if ((boolean_expression) == true) LOGF_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \
+ if ((boolean_expression) == true) LOGF_USER_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \
+ if ((boolean_expression) == true) LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__)
+
+ // Logging with category of "onnxruntime"
+
+#define LOGS_IF(boolean_expression, logger, severity) \
+ LOGS_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGS_DEFAULT_IF(boolean_expression, severity) \
+ LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGS_USER_IF(boolean_expression, logger, severity) \
+ LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGS_USER_DEFAULT_IF(boolean_expression, severity) \
+ LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime)
+
+#define LOGF_IF(boolean_expression, logger, severity, format_str, ...) \
+ LOGF_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+#define LOGF_DEFAULT_IF(boolean_expression, severity, format_str, ...) \
+ LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_IF(boolean_expression, logger, severity, format_str, ...) \
+ LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, \
+ format_str, ##__VA_ARGS__)
+
+#define LOGF_USER_DEFAULT_IF(boolean_expression, severity, format_str, ...) \
+ LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, \
+ format_str, ##__VA_ARGS__)
+
+/*
+
+ Debug verbose logging of caller provided level.
+ Disabled in Release builds.
+ Use the _USER variants for VLOG statements involving user data that may need to be filtered.
+*/
+#define VLOGS(logger, level) \
+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \
+ LOGS_CATEGORY(logger, VERBOSE, "VLOG" #level)
+
+#define VLOGS_USER(logger, level) \
+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \
+ LOGS_USER_CATEGORY(logger, VERBOSE, "VLOG" #level)
+
+#define VLOGF(logger, level, format_str, ...) \
+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \
+ LOGF_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__)
+
+#define VLOGF_USER(logger, level, format_str, ...) \
+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \
+ LOGF_USER_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__)
+
+ // Default logger variants
+#define VLOGS_DEFAULT(level) \
+ VLOGS(::onnxruntime::logging::LoggingManager::DefaultLogger(), level)
+
+#define VLOGS_USER_DEFAULT(level) \
+ VLOGS_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level)
+
+#define VLOGF_DEFAULT(level, format_str, ...) \
+ VLOGF(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__)
+
+#define VLOGF_USER_DEFAULT(level, format_str, ...) \
+ VLOGF_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__)
diff --git a/include/onnxruntime/core/common/logging/severity.h b/include/onnxruntime/core/common/logging/severity.h
new file mode 100644
index 0000000..e43f192
--- /dev/null
+++ b/include/onnxruntime/core/common/logging/severity.h
@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+namespace logging {
+// mild violation of naming convention. the 'k' lets us use token concatenation in the macro
+// ::onnxruntime::Logging::Severity::k##severity. It's not legal to have ::onnxruntime::Logging::Severity::##severity
+// the uppercase makes the LOG macro usage look as expected for passing an enum value as it will be LOGS(logger, ERROR)
+enum class Severity {
+ kVERBOSE = 0,
+ kINFO = 1,
+ kWARNING = 2,
+ kERROR = 3,
+ kFATAL = 4
+};
+
+constexpr const char* SEVERITY_PREFIX = "VIWEF";
+
+} // namespace logging
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/make_string.h b/include/onnxruntime/core/common/make_string.h
new file mode 100644
index 0000000..a6e421a
--- /dev/null
+++ b/include/onnxruntime/core/common/make_string.h
@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// Portions Copyright (c) Microsoft Corporation
+
+#pragma once
+
+#include <locale>
+#include <sstream>
+
+namespace onnxruntime {
+
+namespace detail {
+inline void MakeStringImpl(std::ostringstream& /*ss*/) noexcept {
+}
+
+template <typename T>
+inline void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept {
+ ss << t;
+}
+
+template <typename T, typename... Args>
+inline void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept {
+ MakeStringImpl(ss, t);
+ MakeStringImpl(ss, args...);
+}
+} // namespace detail
+
+/**
+ * Makes a string by concatenating string representations of the arguments.
+ * This version uses the current locale.
+ */
+template <typename... Args>
+std::string MakeString(const Args&... args) {
+ std::ostringstream ss;
+ detail::MakeStringImpl(ss, args...);
+ return ss.str();
+}
+
+/**
+ * Makes a string by concatenating string representations of the arguments.
+ * This version uses std::locale::classic().
+ */
+template <typename... Args>
+std::string MakeStringWithClassicLocale(const Args&... args) {
+ std::ostringstream ss;
+ ss.imbue(std::locale::classic());
+ detail::MakeStringImpl(ss, args...);
+ return ss.str();
+}
+
+// MakeString versions for already-a-string types.
+
+inline std::string MakeString(const std::string& str) {
+ return str;
+}
+
+inline std::string MakeString(const char* cstr) {
+ return cstr;
+}
+
+inline std::string MakeStringWithClassicLocale(const std::string& str) {
+ return str;
+}
+
+inline std::string MakeStringWithClassicLocale(const char* cstr) {
+ return cstr;
+}
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/make_unique.h b/include/onnxruntime/core/common/make_unique.h
new file mode 100644
index 0000000..b401f0d
--- /dev/null
+++ b/include/onnxruntime/core/common/make_unique.h
@@ -0,0 +1,148 @@
+// Copyright 2017 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// -----------------------------------------------------------------------------
+// File: make_unique.h
+// -----------------------------------------------------------------------------
+//
+// This header file contains utility functions for managing the creation and
+// conversion of smart pointers. This file is an extension to the C++
+// standard <memory> library header file.
+/* Modifications Copyright (c) Microsoft. */
+
+#pragma once
+
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <new>
+#include <type_traits>
+#include <utility>
+
+namespace onnxruntime {
+
+template <typename T>
+using remove_extent_t = typename std::remove_extent<T>::type;
+
+namespace memory_internal {
+
+// Traits to select proper overload and return type for `absl::make_unique<>`.
+template <typename T>
+struct MakeUniqueResult {
+ using scalar = std::unique_ptr<T>;
+};
+template <typename T>
+struct MakeUniqueResult<T[]> {
+ using array = std::unique_ptr<T[]>;
+};
+template <typename T, size_t N>
+struct MakeUniqueResult<T[N]> {
+ using invalid = void;
+};
+
+} // namespace memory_internal
+
+// gcc 4.8 has __cplusplus at 201301 but doesn't define make_unique. Other
+// supported compilers either just define __cplusplus as 201103 but have
+// make_unique (msvc), or have make_unique whenever __cplusplus > 201103 (clang)
+#if (__cplusplus > 201103L || defined(_MSC_VER)) && \
+ !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8)
+using std::make_unique;
+#else
+// -----------------------------------------------------------------------------
+// Function Template: make_unique<T>()
+// -----------------------------------------------------------------------------
+//
+// Creates a `std::unique_ptr<>`, while avoiding issues creating temporaries
+// during the construction process. `absl::make_unique<>` also avoids redundant
+// type declarations, by avoiding the need to explicitly use the `new` operator.
+//
+// This implementation of `absl::make_unique<>` is designed for C++11 code and
+// will be replaced in C++14 by the equivalent `std::make_unique<>` abstraction.
+// `absl::make_unique<>` is designed to be 100% compatible with
+// `std::make_unique<>` so that the eventual migration will involve a simple
+// rename operation.
+//
+// For more background on why `std::unique_ptr<T>(new T(a,b))` is problematic,
+// see Herb Sutter's explanation on
+// (Exception-Safe Function Calls)[https://herbsutter.com/gotw/_102/].
+// (In general, reviewers should treat `new T(a,b)` with scrutiny.)
+//
+// Example usage:
+//
+// auto p = make_unique<X>(args...); // 'p' is a std::unique_ptr<X>
+// auto pa = make_unique<X[]>(5); // 'pa' is a std::unique_ptr<X[]>
+//
+// Three overloads of `absl::make_unique` are required:
+//
+// - For non-array T:
+//
+// Allocates a T with `new T(std::forward<Args> args...)`,
+// forwarding all `args` to T's constructor.
+// Returns a `std::unique_ptr<T>` owning that object.
+//
+// - For an array of unknown bounds T[]:
+//
+// `absl::make_unique<>` will allocate an array T of type U[] with
+// `new U[n]()` and return a `std::unique_ptr<U[]>` owning that array.
+//
+// Note that 'U[n]()' is different from 'U[n]', and elements will be
+// value-initialized. Note as well that `std::unique_ptr` will perform its
+// own destruction of the array elements upon leaving scope, even though
+// the array [] does not have a default destructor.
+//
+// NOTE: an array of unknown bounds T[] may still be (and often will be)
+// initialized to have a size, and will still use this overload. E.g:
+//
+// auto my_array = absl::make_unique<int[]>(10);
+//
+// - For an array of known bounds T[N]:
+//
+// `absl::make_unique<>` is deleted (like with `std::make_unique<>`) as
+// this overload is not useful.
+//
+// NOTE: an array of known bounds T[N] is not considered a useful
+// construction, and may cause undefined behavior in templates. E.g:
+//
+// auto my_array = absl::make_unique<int[10]>();
+//
+// In those cases, of course, you can still use the overload above and
+// simply initialize it to its desired size:
+//
+// auto my_array = absl::make_unique<int[]>(10);
+
+// `absl::make_unique` overload for non-array types.
+template <typename T, typename... Args>
+typename memory_internal::MakeUniqueResult<T>::scalar make_unique(
+ Args&&... args) {
+ return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+
+// `absl::make_unique` overload for an array T[] of unknown bounds.
+// The array allocation needs to use the `new T[size]` form and cannot take
+// element constructor arguments. The `std::unique_ptr` will manage destructing
+// these array elements.
+template <typename T>
+typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) {
+ return std::unique_ptr<T>(new typename onnxruntime::remove_extent_t<T>[n]());
+}
+
+// `absl::make_unique` overload for an array T[N] of known bounds.
+// This construction will be rejected.
+template <typename T, typename... Args>
+typename memory_internal::MakeUniqueResult<T>::invalid make_unique(
+ Args&&... /* args */) = delete;
+#endif
+
+}
\ No newline at end of file
diff --git a/include/onnxruntime/core/common/optional.h b/include/onnxruntime/core/common/optional.h
new file mode 100644
index 0000000..ab32cf6
--- /dev/null
+++ b/include/onnxruntime/core/common/optional.h
@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <nonstd/optional.hpp>
+
+namespace onnxruntime {
+
+using nonstd::optional;
+
+#ifndef ORT_NO_EXCEPTIONS
+using nonstd::bad_optional_access;
+#endif
+
+using nonstd::nullopt;
+using nonstd::nullopt_t;
+
+using nonstd::in_place;
+using nonstd::in_place_t;
+
+using nonstd::make_optional;
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/parse_string.h b/include/onnxruntime/core/common/parse_string.h
new file mode 100644
index 0000000..1429859
--- /dev/null
+++ b/include/onnxruntime/core/common/parse_string.h
@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <locale>
+#include <sstream>
+#include <type_traits>
+
+#include "core/common/common.h"
+
+namespace onnxruntime {
+
+/**
+ * Tries to parse a value from an entire string.
+ */
+template <typename T>
+bool TryParseStringWithClassicLocale(const std::string& str, T& value) {
+ if (std::is_integral<T>::value && std::is_unsigned<T>::value) {
+ // if T is unsigned integral type, reject negative values which will wrap
+ if (!str.empty() && str[0] == '-') {
+ return false;
+ }
+ }
+
+ // don't allow leading whitespace
+ if (!str.empty() && std::isspace(str[0], std::locale::classic())) {
+ return false;
+ }
+
+ std::istringstream is{str};
+ is.imbue(std::locale::classic());
+ T parsed_value{};
+
+ const bool parse_successful =
+ is >> parsed_value &&
+ is.get() == std::istringstream::traits_type::eof(); // don't allow trailing characters
+ if (!parse_successful) {
+ return false;
+ }
+
+ value = std::move(parsed_value);
+ return true;
+}
+
+inline bool TryParseStringWithClassicLocale(const std::string& str, std::string& value) {
+ value = str;
+ return true;
+}
+
+inline bool TryParseStringWithClassicLocale(const std::string& str, bool& value) {
+ if (str == "0" || str == "False" || str == "false") {
+ value = false;
+ return true;
+ }
+
+ if (str == "1" || str == "True" || str == "true") {
+ value = true;
+ return true;
+ }
+
+ return false;
+}
+
+/**
+ * Parses a value from an entire string.
+ */
+template <typename T>
+Status ParseStringWithClassicLocale(const std::string& s, T& value) {
+ ORT_RETURN_IF_NOT(TryParseStringWithClassicLocale(s, value), "Failed to parse value: \"", value, "\"");
+ return Status::OK();
+}
+
+/**
+ * Parses a value from an entire string.
+ */
+template <typename T>
+T ParseStringWithClassicLocale(const std::string& s) {
+ T value{};
+ ORT_THROW_IF_ERROR(ParseStringWithClassicLocale(s, value));
+ return value;
+}
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/spin_pause.h b/include/onnxruntime/core/common/spin_pause.h
new file mode 100644
index 0000000..49b71e5
--- /dev/null
+++ b/include/onnxruntime/core/common/spin_pause.h
@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#if defined(_M_AMD64)
+#include <intrin.h>
+#endif
+
+#if defined(__x86_64__)
+#include <xmmintrin.h>
+#endif
+
+namespace onnxruntime {
+
+namespace concurrency {
+
+// Intrinsic to use in spin-loops
+
+inline void SpinPause() {
+#if defined(_M_AMD64) || defined(__x86_64__)
+ _mm_pause();
+#endif
+}
+
+} // namespace concurrency
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/common/status.h b/include/onnxruntime/core/common/status.h
new file mode 100644
index 0000000..c107655
--- /dev/null
+++ b/include/onnxruntime/core/common/status.h
@@ -0,0 +1,191 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// Modifications Copyright (c) Microsoft.
+
+#pragma once
+
+#include <memory>
+#include <ostream>
+#include <string>
+#ifdef _WIN32
+#include <winerror.h>
+#endif
+
+namespace onnxruntime {
+namespace common {
+
+enum StatusCategory {
+ NONE = 0,
+ SYSTEM = 1,
+ ONNXRUNTIME = 2,
+};
+
+/**
+ Error code for ONNXRuntime.
+*/
+enum StatusCode {
+ OK = 0,
+ FAIL = 1,
+ INVALID_ARGUMENT = 2,
+ NO_SUCHFILE = 3,
+ NO_MODEL = 4,
+ ENGINE_ERROR = 5,
+ RUNTIME_EXCEPTION = 6,
+ INVALID_PROTOBUF = 7,
+ MODEL_LOADED = 8,
+ NOT_IMPLEMENTED = 9,
+ INVALID_GRAPH = 10,
+ EP_FAIL = 11
+};
+
+inline const char* StatusCodeToString(StatusCode status) noexcept {
+ switch (status) {
+ case StatusCode::OK:
+ return "SUCCESS";
+ case StatusCode::FAIL:
+ return "FAIL";
+ case StatusCode::INVALID_ARGUMENT:
+ return "INVALID_ARGUMENT";
+ case StatusCode::NO_SUCHFILE:
+ return "NO_SUCHFILE";
+ case StatusCode::NO_MODEL:
+ return "NO_MODEL";
+ case StatusCode::ENGINE_ERROR:
+ return "ENGINE_ERROR";
+ case StatusCode::RUNTIME_EXCEPTION:
+ return "RUNTIME_EXCEPTION";
+ case StatusCode::INVALID_PROTOBUF:
+ return "INVALID_PROTOBUF";
+ case StatusCode::MODEL_LOADED:
+ return "MODEL_LOADED";
+ case StatusCode::NOT_IMPLEMENTED:
+ return "NOT_IMPLEMENTED";
+ case StatusCode::INVALID_GRAPH:
+ return "INVALID_GRAPH";
+ case StatusCode::EP_FAIL:
+ return "EP_FAIL";
+ default:
+ return "GENERAL ERROR";
+ }
+}
+
+#ifdef _WIN32
+inline HRESULT StatusCodeToHRESULT(StatusCode status) noexcept {
+ switch (status)
+ {
+ case StatusCode::OK:
+ return S_OK;
+ case StatusCode::FAIL:
+ return E_FAIL;
+ case StatusCode::INVALID_ARGUMENT:
+ return E_INVALIDARG;
+ case StatusCode::NO_SUCHFILE:
+ return __HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ case StatusCode::NO_MODEL:
+ return __HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
+ case StatusCode::ENGINE_ERROR:
+ return E_FAIL;
+ case StatusCode::RUNTIME_EXCEPTION:
+ return E_FAIL;
+ case StatusCode::INVALID_PROTOBUF:
+ return __HRESULT_FROM_WIN32(ERROR_FILE_CORRUPT);
+ case StatusCode::MODEL_LOADED:
+ return __HRESULT_FROM_WIN32(ERROR_INTERNAL_ERROR);
+ case StatusCode::NOT_IMPLEMENTED:
+ return E_NOTIMPL;
+ case StatusCode::INVALID_GRAPH:
+ return __HRESULT_FROM_WIN32(ERROR_FILE_CORRUPT);
+ case StatusCode::EP_FAIL:
+ return __HRESULT_FROM_WIN32(ERROR_INTERNAL_ERROR);
+ default:
+ return E_FAIL;
+ }
+}
+#endif
+
+class Status {
+ public:
+ Status() noexcept = default;
+
+ Status(StatusCategory category, int code, const std::string& msg);
+
+ Status(StatusCategory category, int code, const char* msg);
+
+ Status(StatusCategory category, int code);
+
+ Status(const Status& other)
+ : state_((other.state_ == nullptr) ? nullptr : new State(*other.state_)) {}
+
+ Status& operator=(const Status& other) {
+ if (state_ != other.state_) {
+ if (other.state_ == nullptr) {
+ state_.reset();
+ } else {
+ state_.reset(new State(*other.state_));
+ }
+ }
+ return *this;
+ }
+
+ Status(Status&&) = default;
+ Status& operator=(Status&&) = default;
+ ~Status() = default;
+
+ bool IsOK() const {
+ return (state_ == nullptr);
+ }
+
+ int Code() const noexcept;
+
+ StatusCategory Category() const noexcept;
+
+ const std::string& ErrorMessage() const noexcept;
+
+ std::string ToString() const;
+
+ bool operator==(const Status& other) const {
+ return (this->state_ == other.state_) || (ToString() == other.ToString());
+ }
+
+ bool operator!=(const Status& other) const {
+ return !(*this == other);
+ }
+
+ static Status OK() {
+ return Status();
+ }
+
+ private:
+ static const std::string& EmptyString() noexcept;
+
+ struct State {
+ State(StatusCategory cat0, int code0, const std::string& msg0)
+ : category(cat0), code(code0), msg(msg0) {}
+
+ State(StatusCategory cat0, int code0, const char* msg0)
+ : category(cat0), code(code0), msg(msg0) {}
+
+ const StatusCategory category;
+ const int code;
+ const std::string msg;
+ };
+
+ // As long as Code() is OK, state_ == nullptr.
+ std::unique_ptr<State> state_;
+};
+
+inline std::ostream& operator<<(std::ostream& out, const Status& status) {
+ return out << status.ToString();
+}
+
+} // namespace common
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/alloc_kind.h b/include/onnxruntime/core/framework/alloc_kind.h
new file mode 100644
index 0000000..4534d08
--- /dev/null
+++ b/include/onnxruntime/core/framework/alloc_kind.h
@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <iosfwd>
+
+namespace onnxruntime {
+// The ml-Values fall into the following categories with respect to their
+// memory management:
+// - inference inputs: owned (allocated and freed) by caller, and is by
+// default read-only by the runtime.
+// - inference outputs: allocated by runtime, ownership transferred to
+// caller. TODO: Make sure this semantics is clear in InferenceSession API.
+// - weights (constant tensors): can be allocated once (statically), and
+// reused by all inference calls within an InferenceSession.
+// - tensor values: The lifetimes of these tensor-values are statically
+// determined, which is used for memory reuse/sharing optimizations. The
+// runtime allocates/frees these values at the right time (as determined
+// by the static allocation plan). Note that this is simplified since we
+// do not try to optimize for "slice" like ops, where we may be able to
+// conditionally reuse memory/data in some cases but not others.
+// Generalizing this is future work.
+
+enum class AllocKind {
+ kNotSet = -1,
+ kAllocate = 0,
+ kReuse = 1,
+ kPreExisting = 2,
+ kAllocateStatically = 3,
+ kAllocateOutput = 4,
+ kShare = 5
+};
+
+std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind);
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h
new file mode 100644
index 0000000..1059c39
--- /dev/null
+++ b/include/onnxruntime/core/framework/allocator.h
@@ -0,0 +1,173 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/common.h"
+#include "core/framework/fence.h"
+#include "core/session/onnxruntime_c_api.h"
+#include "ortdevice.h"
+#include "ortmemoryinfo.h"
+
+// This configures the arena based allocator used by ORT
+// See docs/C_API.md for details on what these mean and how to choose these values
+struct OrtArenaCfg {
+ size_t max_mem; // use 0 to allow ORT to choose the default
+ int arena_extend_strategy; // use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
+ int initial_chunk_size_bytes; // use -1 to allow ORT to choose the default
+ int max_dead_bytes_per_chunk; // use -1 to allow ORT to choose the default
+};
+
+namespace onnxruntime {
+constexpr const char* CPU = "Cpu";
+constexpr const char* CUDA = "Cuda";
+constexpr const char* CUDA_PINNED = "CudaPinned";
+// TODO: Unify the allocator for CUDA and Tensorrt
+constexpr const char* TRT = "Tensorrt";
+constexpr const char* TRT_PINNED = "TensorrtPinned";
+constexpr const char* MIGRAPHX = "MIGraphX";
+constexpr const char* MIGRAPHX_PINNED = "MIGraphXPinned";
+
+constexpr size_t kAllocAlignment = 256;
+
+// forward declaration
+class SessionState;
+
+template <typename T>
+using IAllocatorUniquePtr = std::unique_ptr<T, std::function<void(T*)>>;
+
+class IAllocator {
+ public:
+ IAllocator(const OrtMemoryInfo& info) : memory_info_(info) {}
+ virtual ~IAllocator() = default;
+ /**
+ @remarks Use SafeInt when calculating the size of memory to allocate using Alloc.
+ */
+ virtual void* Alloc(size_t size) = 0;
+ virtual void Free(void* p) = 0;
+ const OrtMemoryInfo& Info() const { return memory_info_; };
+
+ /**
+ optional CreateFence interface, as provider like DML has its own fence
+ */
+ virtual FencePtr CreateFence(const SessionState* /*unused*/) { return nullptr; }
+
+ static bool CalcMemSizeForArray(size_t nmemb, size_t size, size_t* out) noexcept {
+ return CalcMemSizeForArrayWithAlignment(nmemb, size, 0, out);
+ }
+
+ /**
+ * Calculate the memory size for an array. The size is bounds checked using SafeInt.
+ * \tparam alignment must be power of 2
+ * \param nmemb Number of members or elements in the array
+ * \param size Size of each element
+ * \param out Total size required after any alignment is applied
+ * \return true, successful. false, overflow
+ */
+ static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) noexcept ORT_MUST_USE_RESULT;
+
+ /**
+ * https://cwe.mitre.org/data/definitions/190.html
+ * \param alignment must be power of 2
+ * \param nmemb Number of members or elements in the array
+ * \param size Size of each element
+ * \param out Total size required after any alignment is applied
+ * \return true, successful. false, overflow
+ * \remarks This was the original API and was implemented in the header. Replaced with the above version
+ * implemented in the .cc file so that the SafeInt dependency is internal.
+ */
+ template <size_t alignment>
+ static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept ORT_MUST_USE_RESULT;
+
+ /**
+ * allocate memory for an array which has nmemb items of data, each size bytes long
+ */
+ void* AllocArray(size_t nmemb, size_t size) {
+ size_t len;
+ if (!CalcMemSizeForArray(nmemb, size, &len))
+ return nullptr;
+ return Alloc(len);
+ }
+
+ /**
+ * allocate memory for an array which has nmemb items of data, each size bytes long
+ */
+ template <size_t alignment>
+ void* AllocArrayWithAlignment(size_t nmemb, size_t size) {
+ size_t len;
+ if (!CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, &len))
+ return nullptr;
+ return Alloc(len);
+ }
+
+ /**
+ Create a std::unique_ptr that is allocated and freed by the provided IAllocator.
+ @param allocator The allocator.
+ @param count_or_bytes The exact bytes to allocate if T is void, otherwise the number of elements to allocate.
+ @returns std::unique_ptr with allocated memory and deleter.
+ */
+ template <typename T>
+ static IAllocatorUniquePtr<T> MakeUniquePtr(std::shared_ptr<IAllocator> allocator, size_t count_or_bytes) {
+ if (allocator == nullptr) return nullptr;
+ // for now limit to fundamental types. we could support others, but to do so either we or the caller
+ // needs to call the dtor for the objects, for buffers allocated on device we don't have destructor
+ //static_assert(std::is_fundamental<T>::value, "Fundamental type required as no destructors are called.");
+
+ size_t alloc_size = count_or_bytes;
+
+ // if T is not void, 'count_or_bytes' == number of items so allow for that
+ if (!std::is_void<T>::value) {
+ // sizeof(void) isn't valid, but the compiler isn't smart enough to ignore that this line isn't
+ // reachable if T is void. use std::conditional to 'use' void* in the sizeof call
+ if (!CalcMemSizeForArray(count_or_bytes,
+ sizeof(typename std::conditional<std::is_void<T>::value, void*, T>::type),
+ &alloc_size)) return nullptr;
+ }
+
+ return IAllocatorUniquePtr<T>{
+ static_cast<T*>(allocator->Alloc(alloc_size)), // allocate
+ [=](T* ptr) { // capture 'allocator' by value so it's always valid
+ allocator->Free(ptr);
+ }};
+ }
+
+ private:
+ OrtMemoryInfo memory_info_;
+};
+
+template <size_t alignment>
+bool IAllocator::CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept {
+ return CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, out);
+}
+
+class CPUAllocator : public IAllocator {
+ public:
+ explicit CPUAllocator(const OrtMemoryInfo& memory_info) : IAllocator(memory_info) {}
+
+ CPUAllocator() : IAllocator(OrtMemoryInfo(CPU, OrtAllocatorType::OrtDeviceAllocator)) {}
+
+ void* Alloc(size_t size) override;
+ void Free(void* p) override;
+};
+
+#if defined(USE_MIMALLOC_ARENA_ALLOCATOR)
+class MiMallocAllocator : public IAllocator {
+ public:
+ explicit MiMallocAllocator(const OrtMemoryInfo& memory_info) : IAllocator(memory_info) {}
+ MiMallocAllocator() : IAllocator(OrtMemoryInfo(CPU, OrtAllocatorType::OrtDeviceAllocator)) {}
+
+ void* Alloc(size_t size) override;
+ void Free(void* p) override;
+};
+
+#endif
+
+#if defined(USE_MIMALLOC_ARENA_ALLOCATOR)
+using TAllocator = MiMallocAllocator;
+#else
+using TAllocator = CPUAllocator;
+#endif
+
+using AllocatorPtr = std::shared_ptr<IAllocator>;
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/customregistry.h b/include/onnxruntime/core/framework/customregistry.h
new file mode 100644
index 0000000..aafe5a5
--- /dev/null
+++ b/include/onnxruntime/core/framework/customregistry.h
@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/status.h"
+#include "core/common/logging/logging.h"
+#include "core/graph/schema_registry.h"
+#include "core/framework/op_kernel.h"
+#include "core/framework/kernel_def_builder.h"
+#include "core/framework/kernel_registry.h"
+
+namespace onnxruntime {
+
+/**
+ Represents a registry that contains both custom kernels and custom schemas.
+*/
+class CustomRegistry final {
+ public:
+ CustomRegistry() :
+ kernel_registry_(std::make_shared<KernelRegistry>()),
+ opschema_registry_(std::make_shared<onnxruntime::OnnxRuntimeOpSchemaRegistry>()) {}
+
+ /**
+ * Register a kernel definition together with kernel factory method to this session.
+ * If any conflict happened between registered kernel def and built-in kernel def,
+ * registered kernel will have higher priority.
+ * Call this before invoking Initialize().
+ * @return OK if success.
+ */
+ common::Status RegisterCustomKernel(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator);
+
+ common::Status RegisterCustomKernel(KernelCreateInfo&);
+
+ common::Status RegisterOpSet(std::vector<ONNX_NAMESPACE::OpSchema>& schemas, const std::string& domain,
+ int baseline_opset_version, int opset_version);
+
+ const std::shared_ptr<KernelRegistry>& GetKernelRegistry();
+
+ const std::shared_ptr<onnxruntime::OnnxRuntimeOpSchemaRegistry>& GetOpschemaRegistry();
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CustomRegistry);
+ std::shared_ptr<KernelRegistry> kernel_registry_;
+ std::shared_ptr<onnxruntime::OnnxRuntimeOpSchemaRegistry> opschema_registry_;
+
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.h
new file mode 100644
index 0000000..6d4633a
--- /dev/null
+++ b/include/onnxruntime/core/framework/data_types.h
@@ -0,0 +1,1039 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <type_traits>
+#include <map>
+#include <unordered_map>
+
+#include "core/common/common.h"
+#include "core/common/exceptions.h"
+#include "core/framework/endian.h"
+#include "core/graph/onnx_protobuf.h"
+
+struct OrtValue;
+
+namespace ONNX_NAMESPACE {
+class TypeProto;
+} // namespace ONNX_NAMESPACE
+
+namespace onnxruntime {
+/// Predefined registered types
+
+#if !defined(DISABLE_ML_OPS)
+//maps (only used by ML ops)
+using MapStringToString = std::map<std::string, std::string>;
+using MapStringToInt64 = std::map<std::string, int64_t>;
+using MapStringToFloat = std::map<std::string, float>;
+using MapStringToDouble = std::map<std::string, double>;
+using MapInt64ToString = std::map<int64_t, std::string>;
+using MapInt64ToInt64 = std::map<int64_t, int64_t>;
+using MapInt64ToFloat = std::map<int64_t, float>;
+using MapInt64ToDouble = std::map<int64_t, double>;
+#endif
+
+//vectors/sequences
+#if !defined(DISABLE_ML_OPS)
+using VectorMapStringToFloat = std::vector<MapStringToFloat>;
+using VectorMapInt64ToFloat = std::vector<MapInt64ToFloat>;
+#endif
+using VectorString = std::vector<std::string>;
+using VectorInt64 = std::vector<int64_t>;
+
+class DataTypeImpl;
+class TensorTypeBase;
+class SparseTensorTypeBase;
+class SequenceTensorTypeBase;
+class NonTensorTypeBase;
+class PrimitiveDataTypeBase;
+
+// MLFloat16
+union MLFloat16 {
+ uint16_t val;
+
+ explicit MLFloat16(uint16_t x) : val(x) {}
+ MLFloat16() : val(0) {}
+
+ // Taken from https://stackoverflow.com/a/60047308/12627730
+ float AsFloat(uint32_t x) const {
+ float out = 0.0f;
+ std::memcpy(&out, &x, sizeof(x));
+ return out;
+ }
+
+ // Taken from https://stackoverflow.com/a/60047308/12627730
+ uint32_t AsUint(float x) const {
+ uint32_t out = 0;
+ std::memcpy(&out, &x, sizeof(x));
+ return out;
+ }
+
+ float HalfToFloat(const uint16_t x) const {
+ uint16_t half = x;
+ if (endian::native == endian::big) {
+ // Taken from https://stackoverflow.com/a/2182184/12627730
+ half = (x >> 8) | (x << 8);
+ }
+
+ // Taken from https://stackoverflow.com/a/60047308/12627730
+ // IEEE-754 16-bit floating-point format (without infinity): 1-5-10, exp-15, +-131008.0, +-6.1035156E-5,
+ // +-5.9604645E-8, 3.311 digits
+ const uint32_t e = (half & 0x7C00) >> 10; // exponent
+ const uint32_t m = (half & 0x03FF) << 13; // mantissa
+ // evil log2 bit hack to count leading zeros in denormalized format
+ const uint32_t v = AsUint(static_cast<float>(m)) >> 23;
+ uint32_t full = (half & 0x8000) << 16 | (e != 0) * ((e + 112) << 23 | m) |
+ ((e == 0) & (m != 0)) * ((v - 37) << 23 | ((m << (150 - v)) & 0x007FE000)); // sign : normalized : denormalized
+
+ if (endian::native == endian::big) {
+ // Taken from https://stackoverflow.com/a/2182184/12627730
+ full = ((full >> 24) & 0xff) | // move byte 3 to byte 0
+ ((full << 8) & 0xff0000) | // move byte 1 to byte 2
+ ((full >> 8) & 0xff00) | // move byte 2 to byte 1
+ ((full << 24) & 0xff000000); // byte 0 to byte 3
+ }
+
+ return AsFloat(full);
+ }
+
+ operator float() const {
+ return HalfToFloat(val);
+ }
+};
+
+inline bool operator==(const MLFloat16& left, const MLFloat16& right) {
+ return left.val == right.val;
+}
+
+inline bool operator!=(const MLFloat16& left, const MLFloat16& right) {
+ return left.val != right.val;
+}
+
+inline bool operator<(const MLFloat16& left, const MLFloat16& right) {
+ return left.val < right.val;
+}
+
+//BFloat16
+struct BFloat16 {
+ uint16_t val{0};
+ explicit BFloat16() = default;
+ explicit BFloat16(uint16_t v) : val(v) {}
+ explicit BFloat16(float v) {
+ if (endian::native == endian::little) {
+ std::memcpy(&val, reinterpret_cast<char*>(&v) + sizeof(uint16_t), sizeof(uint16_t));
+ } else {
+ std::memcpy(&val, &v, sizeof(uint16_t));
+ }
+ }
+
+ float ToFloat() const {
+ float result;
+ char* const first = reinterpret_cast<char*>(&result);
+ char* const second = first + sizeof(uint16_t);
+ if (endian::native == endian::little) {
+ std::memset(first, 0, sizeof(uint16_t));
+ std::memcpy(second, &val, sizeof(uint16_t));
+ } else {
+ std::memcpy(first, &val, sizeof(uint16_t));
+ std::memset(second, 0, sizeof(uint16_t));
+ }
+ return result;
+ }
+
+ operator float() const {
+ return ToFloat();
+ }
+};
+
+inline void BFloat16ToFloat(const BFloat16* blf, float* flt, size_t size) {
+ auto src = blf;
+ auto d = flt;
+ for (; size != 0; ++src, ++d, --size) {
+ *d = src->ToFloat();
+ }
+}
+
+inline void FloatToBFloat16(const float* flt, BFloat16* blf, size_t size) {
+ auto src = flt;
+ auto d = blf;
+ for (; size != 0; ++src, ++d, --size) {
+ new (d) BFloat16(*src);
+ }
+}
+
+inline bool operator==(const BFloat16& left, const BFloat16& right) {
+ return left.val == right.val;
+}
+
+inline bool operator!=(const BFloat16& left, const BFloat16& right) {
+ return left.val != right.val;
+}
+
+inline bool operator<(const BFloat16& left, const BFloat16& right) {
+ return left.val < right.val;
+}
+
+// DataTypeImpl pointer as unique DataTypeImpl identifier.
+using MLDataType = const DataTypeImpl*;
+// be used with class MLValue
+using DeleteFunc = void (*)(void*);
+using CreateFunc = void* (*)();
+
+/**
+ * \brief Base class for MLDataType
+ *
+ */
+class DataTypeImpl {
+ public:
+ virtual ~DataTypeImpl() = default;
+
+ /**
+ * \brief this API will be used to check type compatibility at runtime
+ *
+ * \param type_proto a TypeProto instance that is constructed for a specific type
+ * will be checked against a TypeProto instance contained within a corresponding
+ * MLDataType instance.
+ */
+ virtual bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const = 0;
+
+ virtual size_t Size() const = 0;
+
+ virtual DeleteFunc GetDeleteFunc() const = 0;
+
+ /**
+ * \brief Retrieves an instance of TypeProto for
+ * a given MLDataType
+ * \returns optional TypeProto. Only ONNX types
+ has type proto, non-ONNX types will return nullptr.
+ */
+ virtual const ONNX_NAMESPACE::TypeProto* GetTypeProto() const = 0;
+
+ virtual bool IsTensorType() const {
+ return false;
+ }
+
+ virtual bool IsTensorSequenceType() const {
+ return false;
+ }
+
+ virtual bool IsSparseTensorType() const {
+ return false;
+ }
+
+ // Returns this if this is of tensor-type and null otherwise
+ virtual const TensorTypeBase* AsTensorType() const {
+ return nullptr;
+ }
+
+ virtual const SequenceTensorTypeBase* AsSequenceTensorBase() const {
+ return nullptr;
+ }
+
+ // Returns this if this is of sparse-tensor-type and null otherwise
+ virtual const SparseTensorTypeBase* AsSparseTensorType() const {
+ return nullptr;
+ }
+
+ virtual const NonTensorTypeBase* AsNonTensorTypeBase() const {
+ return nullptr;
+ }
+
+ // Returns this if this is one of the primitive data types (specialization of PrimitiveDataTypeBase)
+ // and null otherwise
+ virtual const PrimitiveDataTypeBase* AsPrimitiveDataType() const {
+ return nullptr;
+ }
+
+ // Return the type meta that we are using in the runtime.
+ template <typename T>
+ static MLDataType GetType();
+
+ // Return the types for a concrete tensor type, like Tensor_Float
+ template <typename elemT>
+ static MLDataType GetTensorType();
+
+ template <typename elemT>
+ static MLDataType GetSequenceTensorType();
+
+ // Return the MLDataType for a concrete sparse tensor type.
+ template <typename elemT>
+ static MLDataType GetSparseTensorType();
+
+ /**
+ * Convert an ONNX TypeProto to onnxruntime DataTypeImpl.
+ * However, this conversion is lossy. Don't try to use 'this->GetTypeProto()' converting it back.
+ * Even though GetTypeProto() will not have the original information, it will still have enough to correctly
+ * map to MLDataType.
+ * \param proto
+ */
+ static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto);
+
+ static const TensorTypeBase* TensorTypeFromONNXEnum(int type);
+ static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type);
+ static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type);
+
+ static const char* ToString(MLDataType type);
+ static std::vector<std::string> ToString(const std::vector<MLDataType>& types);
+ // Registers ONNX_NAMESPACE::DataType (internalized string) with
+ // MLDataType. DataType is produced by internalizing an instance of
+ // TypeProto contained within MLDataType
+ static void RegisterDataType(MLDataType);
+ static MLDataType GetDataType(const std::string&);
+
+ static const std::vector<MLDataType>& AllTensorTypes();
+ static const std::vector<MLDataType>& AllFixedSizeTensorTypes();
+ static const std::vector<MLDataType>& AllSequenceTensorTypes();
+ static const std::vector<MLDataType>& AllFixedSizeSequenceTensorTypes();
+ static const std::vector<MLDataType>& AllNumericTensorTypes();
+ static const std::vector<MLDataType>& AllIEEEFloatTensorTypes();
+ static const std::vector<MLDataType>& AllFixedSizeTensorExceptHalfTypes();
+ static const std::vector<MLDataType>& AllIEEEFloatTensorExceptHalfTypes();
+ static const std::vector<MLDataType>& AllTensorAndSequenceTensorTypes();
+ static const std::vector<MLDataType>& AllFixedSizeTensorAndSequenceTensorTypes();
+};
+
+std::ostream& operator<<(std::ostream& out, MLDataType data_type);
+
+/*
+ * Type registration helpers
+ */
+namespace data_types_internal {
+/// TensorType helpers
+///
+
+template <typename T>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType();
+
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<float>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT;
+}
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint8_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT8;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int8_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT8;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint16_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int16_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int32_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT32;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int64_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT64;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<std::string>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_STRING;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<bool>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_BOOL;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<MLFloat16>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<double>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint32_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT32;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint64_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT64;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<BFloat16>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16;
+}
+
+// There is a specialization only for one
+// type argument.
+template <typename... Types>
+struct TensorElementTypeSetter {
+ static void SetTensorElementType(ONNX_NAMESPACE::TypeProto&);
+ static void SetMapKeyType(ONNX_NAMESPACE::TypeProto&);
+ static int32_t GetElementType();
+};
+
+/// Is a given type on the list of types?
+/// Accepts a list of types and the first argument is the type
+/// We are checking if it is listed among those that follow
+template <typename T, typename... Types>
+struct IsAnyOf;
+
+/// Two types remaining, end of the list
+template <typename T, typename Tail>
+struct IsAnyOf<T, Tail> : public std::is_same<T, Tail> {
+};
+
+template <typename T, typename H, typename... Tail>
+struct IsAnyOf<T, H, Tail...> {
+ static constexpr bool value = (std::is_same<T, H>::value ||
+ IsAnyOf<T, Tail...>::value);
+};
+
+/// Tells if the specified type is one of fundamental types
+/// that can be contained within a tensor.
+/// We do not have raw fundamental types, rather a subset
+/// of fundamental types is contained within tensors.
+template <typename T>
+struct IsTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, uint16_t, int16_t,
+ int32_t, int64_t, std::string, bool, MLFloat16,
+ double, uint32_t, uint64_t, BFloat16> {
+};
+
+/// Use "IsSparseTensorContainedType<T>::value" to test if a type T
+/// is permitted as the element-type of a sparse-tensor.
+
+template <typename T>
+struct IsSparseTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, uint16_t, int16_t,
+ int32_t, int64_t, bool, MLFloat16,
+ double, uint32_t, uint64_t, BFloat16> {
+};
+
+/// This template's Get() returns a corresponding MLDataType
+/// It dispatches the call to either GetTensorType<>() or
+/// GetType<>()
+template <typename T, bool TensorContainedType>
+struct GetMLDataType;
+
+template <typename T>
+struct GetMLDataType<T, true> {
+ static MLDataType Get() {
+ return DataTypeImpl::GetTensorType<T>();
+ }
+};
+
+template <typename T>
+struct GetMLDataType<T, false> {
+ static MLDataType Get() {
+ return DataTypeImpl::GetType<T>();
+ }
+};
+
+#if !defined(DISABLE_ML_OPS)
+/// MapTypes helper API
+/// K should always be one of the primitive data types
+/// V can be either a primitive type (in which case it is a tensor)
+/// or other preregistered types
+
+void CopyMutableMapValue(const ONNX_NAMESPACE::TypeProto&,
+ ONNX_NAMESPACE::TypeProto&);
+
+template <typename K, typename V>
+struct SetMapTypes {
+ static void Set(ONNX_NAMESPACE::TypeProto& proto) {
+ TensorElementTypeSetter<K>::SetMapKeyType(proto);
+ MLDataType dt = GetMLDataType<V, IsTensorContainedType<V>::value>::Get();
+ const auto* value_proto = dt->GetTypeProto();
+#ifdef ORT_NO_RTTI
+ ORT_ENFORCE(value_proto != nullptr, "expected a registered ONNX type");
+#else
+ ORT_ENFORCE(value_proto != nullptr, typeid(V).name(),
+ " expected to be a registered ONNX type");
+#endif
+ CopyMutableMapValue(*value_proto, proto);
+ }
+};
+#endif
+
+/// Sequence helpers
+///
+// Element type is a primitive type so we set it to a tensor<elemT>
+void CopyMutableSeqElement(const ONNX_NAMESPACE::TypeProto&,
+ ONNX_NAMESPACE::TypeProto&);
+
+template <typename T>
+struct SetSequenceType {
+ static void Set(ONNX_NAMESPACE::TypeProto& proto) {
+ MLDataType dt = GetMLDataType<T, IsTensorContainedType<T>::value>::Get();
+ const auto* elem_proto = dt->GetTypeProto();
+#ifdef ORT_NO_RTTI
+ ORT_ENFORCE(elem_proto != nullptr, "expected a registered ONNX type");
+#else
+ ORT_ENFORCE(elem_proto != nullptr, typeid(T).name(),
+ " expected to be a registered ONNX type");
+#endif
+ CopyMutableSeqElement(*elem_proto, proto);
+ }
+};
+
+/// OpaqueTypes helpers
+///
+void AssignOpaqueDomainName(const char* domain, const char* name,
+ ONNX_NAMESPACE::TypeProto& proto);
+
+} // namespace data_types_internal
+
+/// All tensors base
+class TensorTypeBase : public DataTypeImpl {
+ public:
+ static MLDataType Type();
+
+ /// We first compare type_proto pointers and then
+ /// if they do not match try to account for the case
+ /// where TypeProto was created ad-hoc and not queried from MLDataType
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;
+
+ bool IsTensorType() const override {
+ return true;
+ }
+
+ const TensorTypeBase* AsTensorType() const override {
+ return this;
+ }
+
+ size_t Size() const override;
+
+ DeleteFunc GetDeleteFunc() const override;
+
+ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;
+
+ virtual MLDataType GetElementType() const {
+ // should never reach here.
+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
+ }
+
+ TensorTypeBase(const TensorTypeBase&) = delete;
+ TensorTypeBase& operator=(const TensorTypeBase&) = delete;
+
+ protected:
+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();
+
+ TensorTypeBase();
+ ~TensorTypeBase() override;
+
+ private:
+ struct Impl;
+ Impl* impl_;
+};
+
+/**
+ * \brief Tensor type. This type does not have a C++ type associated with
+ * it at registration time except the element type. One of the types mentioned
+ * above at IsTensorContainedType<> list is acceptable.
+ *
+ * \details
+ * Usage:
+ * ORT_REGISTER_TENSOR(ELEMENT_TYPE)
+ * Currently all of the Tensors irrespective of the dimensions are mapped to Tensor<type>
+ * type. IsCompatible() currently ignores shape.
+ */
+
+template <typename elemT>
+class TensorType : public TensorTypeBase {
+ public:
+ static_assert(data_types_internal::IsTensorContainedType<elemT>::value,
+ "Requires one of the tensor fundamental types");
+
+ static MLDataType Type();
+
+ /// Tensors only can contain basic data types
+ /// that have been previously registered with ONNXRuntime
+ MLDataType GetElementType() const override {
+ return DataTypeImpl::GetType<elemT>();
+ }
+
+ private:
+ TensorType() {
+ using namespace data_types_internal;
+ TensorElementTypeSetter<elemT>::SetTensorElementType(this->mutable_type_proto());
+ }
+};
+
+/// Common base-class for all sparse-tensors (with different element types).
+class SparseTensorTypeBase : public DataTypeImpl {
+ public:
+ static MLDataType Type();
+
+ bool IsSparseTensorType() const override {
+ return true;
+ }
+
+ const SparseTensorTypeBase* AsSparseTensorType() const override {
+ return this;
+ }
+
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;
+
+ size_t Size() const override;
+
+ DeleteFunc GetDeleteFunc() const override;
+
+ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;
+
+ virtual MLDataType GetElementType() const {
+ // should never reach here.
+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
+ }
+
+ SparseTensorTypeBase(const SparseTensorTypeBase&) = delete;
+ SparseTensorTypeBase& operator=(const SparseTensorTypeBase&) = delete;
+
+ protected:
+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();
+
+ SparseTensorTypeBase();
+ ~SparseTensorTypeBase() override;
+
+ private:
+ struct Impl;
+ Impl* impl_;
+};
+
+template <typename elemT>
+class SparseTensorType : public SparseTensorTypeBase {
+ public:
+ static_assert(data_types_internal::IsSparseTensorContainedType<elemT>::value,
+ "Requires one of the sparse-tensor fundamental types");
+
+ static MLDataType Type();
+
+ /// Return a MLDataType representing the element-type
+ MLDataType GetElementType() const override {
+ return DataTypeImpl::GetType<elemT>();
+ }
+
+ private:
+ SparseTensorType() {
+ using namespace data_types_internal;
+ TensorElementTypeSetter<elemT>::SetSparseTensorElementType(mutable_type_proto());
+ }
+};
+
+/**
+ * \brief Provide a specialization for your C++ Non-tensor type
+ * so your implementation FromDataTypeContainer/ToDataTypeContainer
+ * functions correctly. Otherwise you get a default implementation
+ * which may not be what you need/want.
+ *
+ * This class is used to create OrtValue, fetch data from OrtValue via
+ * C/C++ APIs
+ */
+template <class T>
+struct NonTensorTypeConverter {
+ static void FromContainer(MLDataType /*dtype*/, const void* /*data*/, size_t /*data_size*/, OrtValue& /*output*/) {
+ ORT_THROW("Not implemented");
+ }
+ static void ToContainer(const OrtValue& /*input*/, size_t /*data_size*/, void* /*data*/) {
+ ORT_THROW("Not implemented");
+ }
+};
+
+/**
+ * \brief Base type for all non-tensors, maps, sequences and opaques
+ */
+class NonTensorTypeBase : public DataTypeImpl {
+ public:
+ size_t Size() const override = 0;
+
+ DeleteFunc GetDeleteFunc() const override = 0;
+
+ virtual CreateFunc GetCreateFunc() const = 0;
+
+ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;
+
+ const NonTensorTypeBase* AsNonTensorTypeBase() const override {
+ return this;
+ }
+
+ // \brief Override for Non-tensor types to initialize non-tensor CPP
+ // data representation from data. The caller of the interface
+ // should have a shared definition of the data which is used to initialize
+ // CPP data representation. This is used from C API.
+ //
+ // \param data - pointer to a data container structure non_tensor type specific
+ // \param data_size - size of the data container structure, used for rudimentary checks
+ // \param output - reference to a default constructed non-tensor type
+ // \returns OrtValue
+ // \throw if there is an error
+ virtual void FromDataContainer(const void* data, size_t data_size, OrtValue& output) const;
+
+ // \brief Override for Non-tensor types to fetch data from the internal CPP data representation
+ // The caller of the interface should have a shared definition of the data which is used to initialize
+ // CPP data representation. This is used from C API.
+ //
+ // \param input - OrtValue containing data
+ // \param data_size - size of the structure that is being passed for receiving data, used for
+ // validation
+ // \param data - pointer to receiving data structure
+ virtual void ToDataContainer(const OrtValue& input, size_t data_size, void* data) const;
+
+ NonTensorTypeBase(const NonTensorTypeBase&) = delete;
+ NonTensorTypeBase& operator=(const NonTensorTypeBase&) = delete;
+
+ protected:
+ NonTensorTypeBase();
+ ~NonTensorTypeBase() override;
+
+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();
+
+ bool IsMapCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;
+
+ bool IsSequenceCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;
+
+ bool IsOpaqueCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;
+
+ private:
+ struct Impl;
+ Impl* impl_;
+};
+
+// This is where T is the actual CPPRuntimeType
+template <typename T>
+class NonTensorType : public NonTensorTypeBase {
+ private:
+ static void Delete(void* p) {
+ delete static_cast<T*>(p);
+ }
+
+ public:
+ size_t Size() const override {
+ return sizeof(T);
+ }
+
+ DeleteFunc GetDeleteFunc() const override {
+ return &Delete;
+ }
+
+ CreateFunc GetCreateFunc() const override {
+ return []() -> void* { return new T(); };
+ }
+
+ protected:
+ NonTensorType() = default;
+};
+
+#if !defined(DISABLE_ML_OPS)
+/**
+ * \brief MapType. Use this type to register
+ * mapping types.
+ *
+ * \param T - cpp type that you wish to register as runtime MapType
+ *
+ * \details Usage: ORT_REGISTER_MAP(C++Type)
+ * The type is required to have mapped_type and
+ * key_type defined
+ */
+template <typename CPPType>
+class MapType : public NonTensorType<CPPType> {
+ public:
+ static_assert(data_types_internal::IsTensorContainedType<typename CPPType::key_type>::value,
+ "Requires one of the tensor fundamental types as key");
+
+ static MLDataType Type();
+
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {
+ return this->IsMapCompatible(type_proto);
+ }
+
+ private:
+ MapType() {
+ using namespace data_types_internal;
+ SetMapTypes<typename CPPType::key_type, typename CPPType::mapped_type>::Set(this->mutable_type_proto());
+ }
+};
+#endif
+
+/**
+ * \brief SequenceType. Use to register sequence for non-tensor types.
+ *
+ * \param T - CPP type that you wish to register as Sequence
+ * runtime type.
+ *
+ * \details Usage: ORT_REGISTER_SEQ(C++Type)
+ * The type is required to have value_type defined
+ */
+template <typename CPPType>
+class SequenceType : public NonTensorType<CPPType> {
+ public:
+ static MLDataType Type();
+
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {
+ return this->IsSequenceCompatible(type_proto);
+ }
+
+ private:
+ SequenceType() {
+ data_types_internal::SetSequenceType<typename CPPType::value_type>::Set(this->mutable_type_proto());
+ }
+};
+
+/**
+ * \brief SequenceTensorTypeBase serves as a base type class for
+ * Tensor sequences. Akin TensorTypeBase.
+ * Runtime representation is always TensorSeq.
+ */
+class SequenceTensorTypeBase : public DataTypeImpl {
+ public:
+ static MLDataType Type();
+
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;
+
+ bool IsTensorSequenceType() const override {
+ return true;
+ }
+
+ const SequenceTensorTypeBase* AsSequenceTensorBase() const override {
+ return this;
+ }
+
+ virtual MLDataType GetElementType() const {
+ // should never reach here.
+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
+ }
+
+ size_t Size() const override;
+
+ DeleteFunc GetDeleteFunc() const override;
+
+ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;
+
+ SequenceTensorTypeBase(const SequenceTensorTypeBase&) = delete;
+ SequenceTensorTypeBase& operator=(const SequenceTensorTypeBase&) = delete;
+
+ protected:
+ SequenceTensorTypeBase();
+ ~SequenceTensorTypeBase();
+
+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();
+
+ private:
+ struct Impl;
+ Impl* impl_;
+};
+
+/**
+ * \brief SequenceTensorType. Use to register sequence for non-tensor types.
+ *
+ * \param CPPRuntime - We always use TensorSeq
+ *
+ * \param TensorElemType - one of the primitive types
+ *
+ * \details Usage: ORT_REGISTER_SEQ_TENSOR_TYPE()
+ * The type is required to have value_type defined
+ */
+template <typename TensorElemType>
+class SequenceTensorType : public SequenceTensorTypeBase {
+ public:
+ static MLDataType Type();
+
+ /// Return a MLDataType representing the element-type
+ MLDataType GetElementType() const override {
+ return DataTypeImpl::GetType<TensorElemType>();
+ }
+
+ private:
+ SequenceTensorType() {
+ data_types_internal::SetSequenceType<TensorElemType>::Set(this->mutable_type_proto());
+ }
+};
+
+/**
+ * \brief OpaqueType
+ *
+ * \param T - cpp runtume that implements the Opaque type
+ *
+ * \param const char D[] - domain must be extern to be unique
+ *
+ * \param const char N[] - name must be extern to be unique
+ *
+ * \details Only one CPP type can be associated with a particular
+ * OpaqueType registration
+ *
+ */
+template <typename T, const char D[], const char N[]>
+class OpaqueType : public NonTensorType<T> {
+ public:
+ static MLDataType Type();
+
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {
+ return this->IsOpaqueCompatible(type_proto);
+ }
+
+ void FromDataContainer(const void* data, size_t data_size, OrtValue& output) const override {
+ NonTensorTypeConverter<T>::FromContainer(this, data, data_size, output);
+ }
+
+ void ToDataContainer(const OrtValue& input, size_t data_size, void* data) const override {
+ NonTensorTypeConverter<T>::ToContainer(input, data_size, data);
+ }
+
+ private:
+ OpaqueType() {
+ data_types_internal::AssignOpaqueDomainName(D, N, this->mutable_type_proto());
+ }
+};
+
+/**
+ * \brief PrimitiveDataTypeBase
+ * Base class for primitive Tensor contained types
+ *
+ * \details This class contains an integer constant that can be
+ * used for input data type dispatching
+ *
+ */
+class PrimitiveDataTypeBase : public DataTypeImpl {
+ public:
+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto&) const override {
+ return false;
+ }
+
+ const PrimitiveDataTypeBase* AsPrimitiveDataType() const override final {
+ return this;
+ }
+
+ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const final {
+ return nullptr;
+ }
+
+ int32_t GetDataType() const {
+ return data_type_;
+ }
+
+ protected:
+ PrimitiveDataTypeBase() = default;
+
+ void SetDataType(int32_t data_type) {
+ data_type_ = data_type;
+ }
+
+ private:
+ int32_t data_type_;
+};
+
+/**
+ * \brief PrimitiveDataType
+ * Typed specialization for primitive types.
+ * Concrete instances of this class are used by Tensor.
+ *
+ * \param T - primitive data type
+ *
+ */
+template <typename T>
+class PrimitiveDataType : public PrimitiveDataTypeBase {
+ private:
+ static void Delete(void* p) {
+ delete static_cast<T*>(p);
+ }
+
+ public:
+ static MLDataType Type();
+
+ size_t Size() const override {
+ return sizeof(T);
+ }
+
+ DeleteFunc GetDeleteFunc() const override {
+ return &Delete;
+ }
+
+ private:
+ PrimitiveDataType() {
+ this->SetDataType(data_types_internal::TensorElementTypeSetter<T>::GetElementType());
+ }
+};
+
+// Explicit specialization of base class template function
+// is only possible within the enclosing namespace scope,
+// thus a simple way to pre-instantiate a given template
+// at a registration time does not currently work and the macro
+// is needed.
+#define ORT_REGISTER_TENSOR_TYPE(ELEM_TYPE) \
+ template <> \
+ MLDataType TensorType<ELEM_TYPE>::Type() { \
+ static TensorType<ELEM_TYPE> tensor_type; \
+ return &tensor_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetTensorType<ELEM_TYPE>() { \
+ return TensorType<ELEM_TYPE>::Type(); \
+ }
+
+#define ORT_REGISTER_SPARSE_TENSOR_TYPE(ELEM_TYPE) \
+ template <> \
+ MLDataType SparseTensorType<ELEM_TYPE>::Type() { \
+ static SparseTensorType<ELEM_TYPE> tensor_type; \
+ return &tensor_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetSparseTensorType<ELEM_TYPE>() { \
+ return SparseTensorType<ELEM_TYPE>::Type(); \
+ }
+
+#if !defined(DISABLE_ML_OPS)
+#define ORT_REGISTER_MAP(TYPE) \
+ template <> \
+ MLDataType MapType<TYPE>::Type() { \
+ static MapType<TYPE> map_type; \
+ return &map_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetType<TYPE>() { \
+ return MapType<TYPE>::Type(); \
+ }
+#endif
+
+#define ORT_REGISTER_SEQ(TYPE) \
+ template <> \
+ MLDataType SequenceType<TYPE>::Type() { \
+ static SequenceType<TYPE> sequence_type; \
+ return &sequence_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetType<TYPE>() { \
+ return SequenceType<TYPE>::Type(); \
+ }
+
+#define ORT_REGISTER_SEQ_TENSOR_TYPE(ELEM_TYPE) \
+ template <> \
+ MLDataType SequenceTensorType<ELEM_TYPE>::Type() { \
+ static SequenceTensorType<ELEM_TYPE> sequence_tensor_type; \
+ return &sequence_tensor_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetSequenceTensorType<ELEM_TYPE>() { \
+ return SequenceTensorType<ELEM_TYPE>::Type(); \
+ }
+
+#define ORT_REGISTER_PRIM_TYPE(TYPE) \
+ template <> \
+ MLDataType PrimitiveDataType<TYPE>::Type() { \
+ static PrimitiveDataType<TYPE> prim_data_type; \
+ return &prim_data_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetType<TYPE>() { \
+ return PrimitiveDataType<TYPE>::Type(); \
+ }
+
+#define ORT_REGISTER_OPAQUE_TYPE(CPPType, Domain, Name) \
+ template <> \
+ MLDataType OpaqueType<CPPType, Domain, Name>::Type() { \
+ static OpaqueType<CPPType, Domain, Name> opaque_type; \
+ return &opaque_type; \
+ } \
+ template <> \
+ MLDataType DataTypeImpl::GetType<CPPType>() { \
+ return OpaqueType<CPPType, Domain, Name>::Type(); \
+ }
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/data_types_internal.h b/include/onnxruntime/core/framework/data_types_internal.h
new file mode 100644
index 0000000..1f154fc
--- /dev/null
+++ b/include/onnxruntime/core/framework/data_types_internal.h
@@ -0,0 +1,520 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <assert.h>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+#include "core/common/common.h"
+#include "core/framework/data_types.h"
+#include "core/graph/onnx_protobuf.h"
+
+#ifdef _MSC_VER
+#pragma warning(push)
+//TODO: fix the warning in CallableDispatchableRetHelper
+#pragma warning(disable : 4702)
+#endif
+namespace onnxruntime {
+namespace utils {
+
+template <typename T>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED;
+}
+
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<float>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT;
+}
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint8_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT8;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int8_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT8;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint16_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int16_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int32_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT32;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int64_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_INT64;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<std::string>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_STRING;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<bool>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_BOOL;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<MLFloat16>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<double>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint32_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT32;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint64_t>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_UINT64;
+};
+template <>
+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<BFloat16>() {
+ return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16;
+};
+
+ // The following primitives are strongly recommended for switching on tensor input datatypes for
+ // kernel implementations.
+ //
+ // 1) If you need to handle all of the primitive tensor contained datatypes, the best choice would be macros
+ // DispatchOnTensorType or DispatchOnTensorTypeWithReturn. Use inline wrappers so your function can be invoked as function<T>().
+ // 2) if you have a few types, use Tensor.IsDataType<T>()/IsDataTypeString() or use utils::IsPrimitiveDataType<T>()
+ // if you have a standalone MLDatatType with a sequence of if/else statements.
+ // 3) For something in between, we suggest to use CallDispatcher pattern.
+ //
+ // Invoking DataTypeImpl::GetType<T>() for switching on input types is discouraged and should be avoided.
+ // Every primitive type carries with it an integer constant that can be used for quick switching on types.
+
+#define DispatchOnTensorType(tensor_type, function, ...) \
+ switch (tensor_type->AsPrimitiveDataType()->GetDataType()) { \
+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: \
+ function<float>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_BOOL: \
+ function<bool>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: \
+ function<double>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_STRING: \
+ function<std::string>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT8: \
+ function<int8_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT8: \
+ function<uint8_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT16: \
+ function<int16_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT16: \
+ function<uint16_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT32: \
+ function<int32_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT32: \
+ function<uint32_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT64: \
+ function<int64_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT64: \
+ function<uint64_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: \
+ function<MLFloat16>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: \
+ function<BFloat16>(__VA_ARGS__); \
+ break; \
+ default: \
+ ORT_ENFORCE(false, "Unknown tensor type of ", tensor_type); \
+ }
+
+#define DispatchOnTensorTypeWithReturn(tensor_type, retval, function, ...) \
+ switch (tensor_type->AsPrimitiveDataType()->GetDataType()) { \
+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: \
+ retval = function<float>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_BOOL: \
+ retval = function<bool>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: \
+ retval = function<double>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_STRING: \
+ retval = function<std::string>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT8: \
+ retval = function<int8_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT8: \
+ retval = function<uint8_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT16: \
+ retval = function<uint16_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT16: \
+ retval = function<int16_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT32: \
+ retval = function<int32_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT32: \
+ retval = function<uint32_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_INT64: \
+ retval = function<int64_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_UINT64: \
+ retval = function<uint64_t>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: \
+ retval = function<MLFloat16>(__VA_ARGS__); \
+ break; \
+ case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: \
+ retval = function<BFloat16>(__VA_ARGS__); \
+ break; \
+ default: \
+ ORT_ENFORCE(false, "Unknown tensor type of ", tensor_type); \
+ }
+
+////////////////////////////////////////////////////////////////////////////////
+/// Use the following primitives if you have a few types to switch on so you
+// can write a short sequence of if/else statements.
+
+// This is a frequently used check so we make a separate utility function.
+inline bool IsDataTypeString(MLDataType dt_type) {
+ auto prim_type = dt_type->AsPrimitiveDataType();
+ return (prim_type != nullptr && prim_type->GetDataType() == ONNX_NAMESPACE::TensorProto_DataType_STRING);
+}
+
+// Test if MLDataType is a concrete type of PrimitiveDataTypeBase
+// and it is T
+template <class T>
+inline bool IsPrimitiveDataType(MLDataType dt_type) {
+ auto prim_type = dt_type->AsPrimitiveDataType();
+ return (prim_type != nullptr && prim_type->GetDataType() == ToTensorProtoElementType<T>());
+}
+
+// Use after AsPrimitiveDataType() is successful
+// Check if PrimitiveDataTypeBase is of type T
+template <class T>
+inline bool IsPrimitiveDataType(const PrimitiveDataTypeBase* prim_type) {
+ assert(prim_type != nullptr);
+ return prim_type->GetDataType() == ToTensorProtoElementType<T>();
+}
+
+// This implementation contains a workaround for GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47226
+// GCC until very recently does not support template parameter pack expansion within lambda context.
+namespace mltype_dispatcher_internal {
+// T - type handled by this helper
+struct CallableDispatchableHelper {
+ int32_t dt_type_; // Type currently dispatched
+ size_t called_;
+
+ explicit CallableDispatchableHelper(int32_t dt_type) noexcept : dt_type_(dt_type), called_(0) {}
+
+ // Must return integer to be in a expandable context
+ template <class T, class Fn, class... Args>
+ int Invoke(Fn&& fn, Args&&... args) {
+ if (utils::ToTensorProtoElementType<T>() == dt_type_) {
+ std::forward<Fn>(fn)(std::forward<Args>(args)...);
+ ++called_;
+ }
+ return 0;
+ }
+};
+
+// Default policy is to throw with no return type.
+template <class Ret>
+struct UnsupportedTypeDefaultPolicy {
+ Ret operator()(int32_t dt_type) const {
+ ORT_THROW("Unsupported data type: ", dt_type);
+ }
+};
+
+// Helper with the result type
+template <class Ret, class UnsupportedPolicy = UnsupportedTypeDefaultPolicy<Ret>>
+struct CallableDispatchableRetHelper {
+ int32_t dt_type_; // Type currently dispatched
+ size_t called_;
+ Ret result_;
+
+ explicit CallableDispatchableRetHelper(int32_t dt_type) noexcept : dt_type_(dt_type), called_(0), result_() {}
+
+ Ret Get() {
+ // See if there were multiple invocations.It is a bug.
+ ORT_ENFORCE(called_ < 2, "Check for duplicate types in MLTypeCallDispatcherRet");
+ // No type was invoked
+ if (called_ == 0) {
+ result_ = UnsupportedPolicy()(dt_type_);
+ }
+ return result_;
+ }
+
+ // Must return integer to be in a expandable context
+ template <class T, class Fn, class... Args>
+ int Invoke(Fn&& fn, Args&&... args) {
+ if (utils::ToTensorProtoElementType<T>() == dt_type_) {
+ result_ = std::forward<Fn>(fn)(std::forward<Args>(args)...);
+ ++called_;
+ }
+ return 0;
+ }
+};
+
+} // namespace mltype_dispatcher_internal
+
+// This class helps to efficiently dispatch calls for templated
+// kernel implementation functions that has no return value.
+// If your implementation function must return a value such as Status
+// Use MLTypeCallDispatcherRet class.
+//
+// The first template parameter is a template<T> struct/class functor
+// that must implement operator() with arbitrary number of arguments
+// and void return turn. It must return Ret type if you are using MLTypeCallDispatcherRet.
+// Fn must be default constructible.
+//
+// Types is a type list that are supported by this kernel implementation.
+// There should be no duplicate types. An exception will be thrown if there
+// a duplicate.
+//
+// The constructor accepts an enum that is obtained from
+// input_tensor->DataType()->AsPrimitiveType()->GetDataType().
+// Fn will be called only once the type designated by dt_type value.
+// If current dt_type is not handled, the Dispatcher will throw an exception.
+//
+template <template <typename> class Fn, typename... Types>
+class MLTypeCallDispatcher {
+ int32_t dt_type_;
+
+ public:
+ explicit MLTypeCallDispatcher(int32_t dt_type) noexcept : dt_type_(dt_type) {}
+
+ template <typename... Args>
+ void Invoke(Args&&... args) const {
+ mltype_dispatcher_internal::CallableDispatchableHelper helper(dt_type_);
+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};
+ ORT_UNUSED_PARAMETER(results);
+ ORT_ENFORCE(helper.called_ < 2, "Check for duplicate types in MLTypeCallDispatcher");
+ ORT_ENFORCE(helper.called_ == 1, "Unsupported data type: ", dt_type_);
+ }
+};
+
+// Version of the MLTypeDispatcher with a return type.
+// Return type of Fn must return type convertible to Ret
+// The value of the return type will be the return value
+// of the function for type T which was specified for execution.
+template <class Ret, template <typename> class Fn, typename... Types>
+class MLTypeCallDispatcherRet {
+ int32_t dt_type_;
+
+ public:
+ explicit MLTypeCallDispatcherRet(int32_t dt_type) noexcept : dt_type_(dt_type) {}
+
+ template <typename... Args>
+ Ret Invoke(Args&&... args) const {
+ mltype_dispatcher_internal::CallableDispatchableRetHelper<Ret> helper(dt_type_);
+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};
+ ORT_UNUSED_PARAMETER(results);
+ return helper.Get();
+ }
+
+ template <class UnsupportedPolicy, typename... Args>
+ Ret InvokeWithUnsupportedPolicy(Args&&... args) const {
+ mltype_dispatcher_internal::CallableDispatchableRetHelper<Ret, UnsupportedPolicy> helper(dt_type_);
+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};
+ ORT_UNUSED_PARAMETER(results);
+ return helper.Get();
+ }
+};
+
+// Version of the MLTypeDispatcher that has an input type which is passed through ('carried')
+// as the first type parameter in the call to Fn when dispatching.
+template <typename TCarried, template <typename, typename> class Fn, typename... Types>
+class MLTypeCallDispatcherWithCarriedType {
+ int32_t dt_type_;
+
+ public:
+ explicit MLTypeCallDispatcherWithCarriedType(int32_t dt_type) noexcept : dt_type_(dt_type) {}
+
+ template <typename... Args>
+ void Invoke(Args&&... args) const {
+ mltype_dispatcher_internal::CallableDispatchableHelper helper(dt_type_);
+ int results[] = {0, helper.template Invoke<Types>(Fn<TCarried, Types>(), std::forward<Args>(args)...)...};
+ ORT_UNUSED_PARAMETER(results);
+ ORT_ENFORCE(helper.called_ < 2, "Check for duplicate types in MLTypeCallDispatcher");
+ ORT_ENFORCE(helper.called_ == 1, "Unsupported data type: ", dt_type_);
+ }
+};
+
+namespace data_types_internal {
+
+enum class ContainerType : uint16_t {
+ kUndefined = 0,
+ kTensor = 1,
+ kMap = 2,
+ kSequence = 3,
+ kOpaque = 4
+};
+
+class TypeNode {
+ // type_ is a TypeProto value case enum
+ // that may be a kTypeTensor, kTypeMap, kTypeSequence
+ // prim_type_ is a TypeProto_DataType enum that has meaning
+ // - for Tensor then prim_type_ is the contained type
+ // - for Map prim_type is the key type. Next entry describes map value
+ // - For sequence prim_type_ is not used and has no meaning. Next entry
+ // describes the value for the sequence
+ // Tensor is always the last entry as it describes a contained primitive type.
+ ContainerType type_;
+ uint16_t prim_type_;
+
+ public:
+ TypeNode(ContainerType type, int32_t prim_type) noexcept {
+ type_ = type;
+ prim_type_ = static_cast<uint16_t>(prim_type);
+ }
+
+ bool IsType(ContainerType type) const noexcept {
+ return type_ == type;
+ }
+
+ bool IsPrimType(int32_t prim_type) const noexcept {
+ return prim_type_ == static_cast<uint16_t>(prim_type);
+ }
+};
+
+} // namespace data_types_internal
+
+////////////////////////////////////////////////////////////////////
+/// Provides generic interface to test whether MLDataType is a Sequence,
+/// Map or an Opaque type including arbitrary recursive definitions
+/// without querying DataTypeImpl::GetType<T> for all known complex types
+
+// T is a sequence contained element type
+// If returns true then we know that the runtime
+// representation is std::vector<T>
+// T itself can be a runtime representation of another
+// sequence, map, opaque type or a tensor
+//
+// That is it can be std::vector or a std::map
+// If T is a primitive type sequence is tested whether it contains
+// tensors of that type
+//
+// If T is an opaque type, then it is only tested to be opaque but not exactly
+// a specific opaque type. To Test for a specific Opaque type use IsOpaqueType() below
+//
+// This class examines the supplied MLDataType and records
+// its information in a vector so any subsequent checks for Sequences and Maps
+// are quick.
+class ContainerChecker {
+ using Cont = std::vector<data_types_internal::TypeNode>;
+ Cont types_;
+
+ // Default IsContainerOfType is for Opaque type
+ template <class T>
+ struct IsContainerOfType {
+ static bool check(const Cont& c, size_t index) {
+ if (index >= c.size()) {
+ return false;
+ }
+ return c[index].IsType(data_types_internal::ContainerType::kOpaque);
+ }
+ };
+
+ // Handles the case where sequence element is also a sequence
+ template <class T>
+ struct IsContainerOfType<std::vector<T>> {
+ static bool check(const Cont& c, size_t index) {
+ if (index >= c.size()) {
+ return false;
+ }
+ if (c[index].IsType(data_types_internal::ContainerType::kSequence)) {
+ ORT_ENFORCE(++index < c.size(), "Sequence is missing type entry for its element");
+ constexpr int32_t prim_type = ToTensorProtoElementType<T>();
+ // Check if this is a primitive type and it matches
+ if (prim_type != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED) {
+ return c[index].IsType(data_types_internal::ContainerType::kTensor) &&
+ c[index].IsPrimType(prim_type);
+ } else {
+ // T is not primitive, check next entry for non-primitive proto
+ return IsContainerOfType<T>::check(c, index);
+ }
+ }
+ return false;
+ }
+ };
+
+ template <class K, class V>
+ struct IsContainerOfType<std::map<K, V>> {
+ static bool check(const Cont& c, size_t index) {
+ static_assert(ToTensorProtoElementType<K>() != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED,
+ "Map Key can not be a non-primitive type");
+ if (index >= c.size()) {
+ return false;
+ }
+ if (!c[index].IsType(data_types_internal::ContainerType::kMap)) {
+ return false;
+ }
+ constexpr int32_t key_type = ToTensorProtoElementType<K>();
+ if (!c[index].IsPrimType(key_type)) {
+ return false;
+ }
+ ORT_ENFORCE(++index < c.size(), "Map is missing type entry for its value");
+ constexpr int32_t val_type = ToTensorProtoElementType<V>();
+ if (val_type != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED) {
+ return c[index].IsType(data_types_internal::ContainerType::kTensor) &&
+ c[index].IsPrimType(val_type);
+ }
+ return IsContainerOfType<V>::check(c, index);
+ }
+ };
+
+ public:
+ explicit ContainerChecker(MLDataType);
+ ~ContainerChecker() = default;
+
+ bool IsMap() const noexcept {
+ assert(!types_.empty());
+ return types_[0].IsType(data_types_internal::ContainerType::kMap);
+ }
+
+ bool IsSequence() const noexcept {
+ assert(!types_.empty());
+ return types_[0].IsType(data_types_internal::ContainerType::kSequence);
+ }
+
+ template <class T>
+ bool IsSequenceOf() const {
+ assert(!types_.empty());
+ return IsContainerOfType<std::vector<T>>::check(types_, 0);
+ }
+
+ template <class K, class V>
+ bool IsMapOf() const {
+ assert(!types_.empty());
+ return IsContainerOfType<std::map<K, V>>::check(types_, 0);
+ }
+};
+
+bool IsOpaqueType(MLDataType ml_type, const char* domain, const char* name);
+
+} // namespace utils
+} // namespace onnxruntime
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
diff --git a/include/onnxruntime/core/framework/endian.h b/include/onnxruntime/core/framework/endian.h
new file mode 100644
index 0000000..629fb78
--- /dev/null
+++ b/include/onnxruntime/core/framework/endian.h
@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+
+// the semantics of this enum should match std::endian from C++20
+enum class endian {
+#if defined(_WIN32)
+ little = 0,
+ big = 1,
+ native = little,
+#elif defined(__GNUC__) || defined(__clang__)
+ little = __ORDER_LITTLE_ENDIAN__,
+ big = __ORDER_BIG_ENDIAN__,
+ native = __BYTE_ORDER__,
+#else
+#error onnxruntime::endian is not implemented in this environment.
+#endif
+};
+
+static_assert(
+ endian::native == endian::little || endian::native == endian::big,
+ "Only little-endian or big-endian native byte orders are supported.");
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.h
new file mode 100644
index 0000000..3c877f2
--- /dev/null
+++ b/include/onnxruntime/core/framework/execution_provider.h
@@ -0,0 +1,278 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#ifndef PROVIDER_BRIDGE_PROVIDER
+#include <map>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "core/common/status.h"
+#include "core/common/logging/logging.h"
+#include "core/framework/tensor.h"
+#include "core/framework/data_transfer.h"
+
+namespace onnxruntime {
+
+class GraphViewer;
+class Node;
+struct ComputeCapability;
+class KernelRegistry;
+class KernelRegistryManager;
+} // namespace onnxruntime
+#endif
+
+#include "core/framework/provider_options.h"
+#include "core/framework/func_api.h"
+
+namespace onnxruntime {
+
+/**
+ Logical device representation.
+*/
+using AllocatorMap = std::map<int, AllocatorPtr>;
+using MemoryInfoSet = std::set<OrtMemoryInfo>;
+
+// if we are export the fused function to dll, the function will still in the same binary as onnxruntime
+// use std function to give execution provider some chance to capture some state.
+using CreateFunctionStateFunc = std::function<int(ComputeContext*, FunctionState*)>;
+using ComputeFunc = std::function<Status(FunctionState, const OrtApi*, OrtKernelContext*)>;
+using DestroyFunctionStateFunc = std::function<void(FunctionState)>;
+
+struct NodeComputeInfo {
+ CreateFunctionStateFunc create_state_func;
+ ComputeFunc compute_func;
+ DestroyFunctionStateFunc release_state_func;
+};
+
+class IExecutionProvider {
+ protected:
+ IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false)
+ : type_{type} {
+ if (use_metadef_id_creator) {
+ metadef_id_generator_ = onnxruntime::make_unique<ModelMetadefIdGenerator>();
+ }
+ }
+
+ public:
+ virtual ~IExecutionProvider() = default;
+
+ /**
+ Get all IAllocators for <*this> execution provider.
+ */
+ const std::vector<AllocatorPtr>& GetAllocators() const {
+ return allocator_list_;
+ }
+
+ /**
+ * Get an allocator with specified device id and MemType. Return nullptr if it doesn't exist
+ */
+ virtual AllocatorPtr GetAllocator(int id, OrtMemType mem_type) const;
+
+ /**
+ * Returns a data transfer object that implements methods to copy to and
+ * from this device.
+ * If no copy is required for the successful operation of this provider,
+ * return a nullptr.
+ */
+ virtual std::unique_ptr<onnxruntime::IDataTransfer> GetDataTransfer() const {
+ return nullptr;
+ }
+
+ /**
+ Get execution provider's capability for the specified <graph>.
+ Return a bunch of IndexedSubGraphs <*this> execution provider can run if
+ the sub-graph contains only one node or can fuse to run if the sub-graph
+ contains more than one node. The node indexes contained in sub-graphs may
+ have overlap, and it's ONNXRuntime's responsibility to do the partition
+ and decide whether a node will be assigned to <*this> execution provider.
+ */
+ virtual std::vector<std::unique_ptr<ComputeCapability>>
+ GetCapability(const onnxruntime::GraphViewer& graph_viewer,
+ const std::vector<const KernelRegistry*>& kernel_registries) const;
+
+ /**
+ Get kernel registry per execution provider type.
+ The KernelRegistry share pointer returned is shared across sessions.
+
+ NOTE: this is a tricky but final solution to achieve following goals,
+ 1. The execution provider type based kernel registry should be shared
+ across sessions.
+ Only one copy of this kind of kernel registry exists in ONNXRuntime
+ with multiple sessions/models.
+ 2. Adding an execution provider into ONNXRuntime does not need to touch ONNXRuntime
+ frameowrk/session code.
+ 3. onnxruntime (framework/session) does not depend on any specific
+ execution provider lib.
+ */
+ virtual std::shared_ptr<KernelRegistry> GetKernelRegistry() const { return nullptr; }
+
+ /**
+ Get the device id of current execution provider
+ */
+ virtual int GetDeviceId() const { return -1; };
+
+ /**
+ Get execution provider's configuration options.
+ */
+ virtual ProviderOptions GetProviderOptions() const { return {}; }
+
+ /**
+ Returns an opaque handle whose exact type varies based on the provider
+ and is interpreted accordingly by the corresponding kernel implementation.
+ For Direct3D operator kernels, this may return an IUnknown supporting
+ QueryInterface to ID3D12GraphicsCommandList1.
+ */
+ virtual const void* GetExecutionHandle() const noexcept {
+ return nullptr;
+ }
+
+ /**
+ @return type of the execution provider; should match that set in the node
+ through the SetExecutionProvider API. Example valid return values are:
+ kCpuExecutionProvider, kCudaExecutionProvider
+ */
+ const std::string& Type() const { return type_; }
+
+ /**
+ Blocks until the device has completed all preceding requested tasks.
+ Currently this is primarily used by the IOBinding object to ensure that all
+ inputs have been copied to the device before execution begins.
+ */
+ virtual common::Status Sync() const { return Status::OK(); }
+
+ /**
+ Called when InferenceSession::Run started
+ NOTE that due to async execution in provider, the actual work of previous
+ Run may not be finished on device This function should be regarded as the
+ point after which a new Run would start to submit commands from CPU
+ */
+ virtual common::Status OnRunStart() { return Status::OK(); }
+
+ /**
+ Called when InferenceSession::Run ended
+ NOTE that due to async execution in provider, the actual work of this Run
+ may not be finished on device This function should be regarded as the point
+ that all commands of current Run has been submmited by CPU
+ */
+ virtual common::Status OnRunEnd() { return Status::OK(); }
+
+ /**
+ Called when session creation is complete
+ This provides an opportunity for execution providers to optionally synchronize and
+ clean up its temporary resources to reduce memory and ensure the first run is fast.
+ */
+ virtual common::Status OnSessionInitializationEnd() { return Status::OK(); }
+
+ void InsertAllocator(AllocatorPtr allocator);
+ void ReplaceAllocator(AllocatorPtr allocator);
+
+ // creation of a fused node is not supported in a minimal build, so any EP enabled in that scenario must support
+ // compilation via GraphViewer instances.
+#if !defined(ORT_MINIMAL_BUILD)
+ /**
+ Given a list of fused_node, return create_state/compute/release_state func for each node.
+ */
+ virtual common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
+ std::vector<NodeComputeInfo>& node_compute_funcs);
+
+ /**
+ Given a list of fused_node, return a dll that expose functions for each node.
+ For each node, there should be three symbols:
+ Create_State_${node_name}
+ Compute_${node_name}
+ Release_State_${node_name}
+ */
+ virtual common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,
+ std::string& dll_path);
+
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
+ struct FusedNodeAndGraph {
+ const std::reference_wrapper<onnxruntime::Node> fused_node;
+ // GraphViewer that filters the full graph to the nodes that are covered by 'node'
+ const std::reference_wrapper<GraphViewer> filtered_graph;
+ };
+
+ /**
+ Given a collection of fused Nodes and the respective GraphViewer instance for the nodes that were fused,
+ return create_state/compute/release_state func for each node.
+ @remarks This is an optional interface that is only needed if the execution provider compiles nodes
+ in a scenario involving the minimal build. i.e. on a mobile or embedded device with ORT format model.
+
+ Do NOT cache the GraphViewer in FusedNodeAndGraph.filtered_graph in any of the NodeComputeInfo functions
+ as it is only valid for the duration of the call to Compile.
+ */
+ virtual common::Status Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,
+ std::vector<NodeComputeInfo>& node_compute_funcs);
+#endif
+
+ // Fusion approach that is suppported
+ enum class FusionStyle {
+ // The node fusion will create an onnxruntime::Function based Node that contains a completely new Graph instance
+ // in the Node body. The original nodes and initializers are copied to the new Graph instance in Function::Body().
+ // A GraphProto can be produced from the Node body.
+ Function,
+
+ // The node fusion will create a new Node that defines the inputs and outputs using the IndexedSubGraph
+ // that GetCapability returned. The Node will not be onnxruntime::Function based so will have no Body().
+ // Instead a GraphViewer that filters the full Graph to the fused Nodes will be created.
+ // This is significantly cheaper as it doesn't incur the cost of creating a new Graph instance,
+ // and can be supported in a minimal build.
+ FilteredGraphViewer
+ };
+
+ virtual FusionStyle GetFusionStyle() const {
+ // existing EPs use this mode so default to it.
+ // newer EPs that can use the cheaper approach, or need to run in a minimal build, should override to return
+ // FilteredGraphViewer
+ return FusionStyle::Function;
+ }
+
+ void SetLogger(const logging::Logger* logger) {
+ logger_ = logger;
+ }
+
+ const logging::Logger* GetLogger() const {
+ return logger_;
+ }
+
+ /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance.
+ The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models.
+ @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph.
+ @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model.
+ This is created using the model path if available,
+ or the model input names and the output names from all nodes in the main graph.
+ @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches
+ compiled kernels, so the name must be unique and deterministic across models and sessions.
+ NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and
+ virtual, and ModelMetadefIdGenerator but be defined in the header as well.
+ */
+ virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const;
+
+ private:
+ const std::string type_;
+ AllocatorMap allocators_;
+ MemoryInfoSet mem_info_set_; // to ensure only allocators with unique OrtMemoryInfo are registered in the provider.
+ //It will be set when this object is registered to a session
+ const logging::Logger* logger_ = nullptr;
+ // convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time
+ // contains the same instances as allocators_
+ std::vector<AllocatorPtr> allocator_list_;
+
+ // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across
+ // multiple sessions.
+ class ModelMetadefIdGenerator {
+ public:
+ int GenerateId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash);
+
+ private:
+ std::unordered_map<uint64_t, int64_t> main_graph_hash_; // map graph instance hash to model contents hash
+ std::unordered_map<int64_t, int> model_metadef_id_; // current unique id for model
+ };
+
+ std::unique_ptr<ModelMetadefIdGenerator> metadef_id_generator_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/fence.h b/include/onnxruntime/core/framework/fence.h
new file mode 100644
index 0000000..e8408d1
--- /dev/null
+++ b/include/onnxruntime/core/framework/fence.h
@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+
+using ProviderType = const std::string&;
+
+/*
+ We use a simple fence mechanism for async compute. Assumptions in this fence mechanism:
+ * Execution provider command queues, which execute in the same order of submit
+ * No fence needed for kernels within one execution provider command queue
+ * Fence is used to synchronize between command queues, and execution providers
+
+ Fence usage:
+ 1. Fence object would be created by allocation planer for input/output when KernelDef::ExecQueueId() is not zero
+ 2. If fence object exists, executor would call BeforeUsingAs* prior to kernel::Compute(), and AfterUsedAs* afterwards
+*/
+class IFence {
+ public:
+ virtual ~IFence() = default;
+
+ /**
+ Called by executor before OrtValue is used as input in a compute kernel in provider_type and exec queue_id
+ This should wait in the specified provider's exec queue for previous write to OrtValue to finish
+ */
+ virtual void BeforeUsingAsInput(onnxruntime::ProviderType provider_type, int queue_id) = 0;
+
+ /**
+ Called by executor before OrtValue is used as output in a compute kernel in provider_type and exec queue_id
+ This should wait in the specified provider's exec queue for previous read to OrtValue to finish
+ */
+ virtual void BeforeUsingAsOutput(onnxruntime::ProviderType provider_type, int queue_id) = 0;
+
+ /**
+ Called by executor after OrtValue is used as input in a compute kernel in provider_type and exec queue_id
+ This should update the read fence of the MLValue
+ */
+ virtual void AfterUsedAsInput(int queue_id) = 0;
+
+ /**
+ Called by executor after OrtValue is used as output in a compute kernel in provider_type and exec queue_id
+ This should update the write fence of the MLValue
+ */
+ virtual void AfterUsedAsOutput(int queue_id) = 0;
+
+ /**
+ Called by executor before release OrtValue to see whether async data read is finished or not. This is non-blocking.
+ */
+ virtual bool CanRelease() = 0;
+};
+using Fence_t = IFence*;
+using FencePtr = std::shared_ptr<IFence>;
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/framework_common.h b/include/onnxruntime/core/framework/framework_common.h
new file mode 100644
index 0000000..dd0dea9
--- /dev/null
+++ b/include/onnxruntime/core/framework/framework_common.h
@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "run_options.h"
+
+namespace onnxruntime { // forward declarations
+class Model;
+class GraphTransformer;
+class NodeArg;
+} // namespace onnxruntime
+
+namespace onnxruntime {
+using InputDefList = std::vector<const onnxruntime::NodeArg*>;
+using OutputDefList = std::vector<const onnxruntime::NodeArg*>;
+
+using NameMLValMap = std::unordered_map<std::string, OrtValue>;
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/func_api.h b/include/onnxruntime/core/framework/func_api.h
new file mode 100644
index 0000000..93b52d8
--- /dev/null
+++ b/include/onnxruntime/core/framework/func_api.h
@@ -0,0 +1,27 @@
+#pragma once
+#include "core/common/status.h"
+using onnxruntime::common::Status; // TODO: Needed by WinML, but shouldn't be put into the global namespace like this
+
+namespace onnxruntime {
+
+// AllocateFunc(void* handle, size_t alignment, size_t size)
+using AllocateFunc = void* (*)(void*, size_t, size_t);
+using DestroyFunc = void (*)(void*, void*);
+using AllocatorHandle = void*;
+
+typedef struct {
+ //right now we only include allocation for host memory
+ AllocateFunc allocate_func;
+ DestroyFunc release_func;
+ AllocatorHandle allocator_handle;
+ const char* node_name;
+} ComputeContext;
+
+using FunctionState = void*;
+// take the ComputeContext, and create a function state.
+using CreateFunctionStateC = int (*)(ComputeContext*, FunctionState*);
+// pass in the function state and input/output tensors, perform compute and return status
+using ComputeFuncC = common::Status (*)(FunctionState, const OrtApi*, OrtKernelContext*);
+// release the function state.
+using DestroyFunctionStateC = void (*)(FunctionState);
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h
new file mode 100644
index 0000000..63c9269
--- /dev/null
+++ b/include/onnxruntime/core/framework/kernel_def_builder.h
@@ -0,0 +1,339 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <limits.h>
+
+#include "core/common/common.h"
+#include "core/common/optional.h"
+#include "core/graph/basic_types.h"
+#include "core/framework/data_types.h"
+#include "core/framework/allocator.h"
+
+namespace onnxruntime {
+class KernelDefBuilder;
+
+typedef std::map<size_t, OrtMemType> MemTypeMap;
+
+class KernelDef {
+ private:
+ // note that input/output might be on CPU implicitly when the node is from CPU execution provider
+ static inline bool MemTypeOnCpuExplicitly(OrtMemType mem_type) {
+ return mem_type == OrtMemTypeCPUInput || mem_type == OrtMemTypeCPUOutput;
+ }
+
+ public:
+ explicit KernelDef() = default;
+
+ const std::string& OpName() const {
+ return op_name_;
+ }
+
+ const std::string& Domain() const {
+ return op_domain_;
+ }
+
+ void SinceVersion(/*out*/ int* start, /*out*/ int* end) const {
+ *start = op_since_version_start_;
+ *end = op_since_version_end_;
+ }
+
+#ifdef onnxruntime_PYBIND_EXPORT_OPSCHEMA
+ const std::pair<int, int> SinceVersion() const {
+ return std::pair<int, int>(op_since_version_start_, op_since_version_end_);
+ }
+#endif
+
+ onnxruntime::ProviderType Provider() const {
+ return provider_type_;
+ }
+
+ const std::map<std::string, std::vector<MLDataType>>& TypeConstraints() const {
+ return type_constraints_;
+ }
+
+ const std::vector<std::pair<int, int>>& MayInplace() const {
+ return inplace_map_;
+ }
+
+ const std::vector<std::pair<int, int>>& Alias() const {
+ return alias_map_;
+ }
+
+ const optional<std::pair<int, int>>& VariadicAlias() const {
+ return variadic_alias_offsets_;
+ }
+
+ OrtMemType InputMemoryType(size_t input_index) const {
+ auto it = input_memory_type_args_.find(input_index);
+ if (it == input_memory_type_args_.end())
+ return default_inputs_mem_type_;
+ return it->second;
+ }
+
+ bool IsInputOnCpu(size_t input_index) const { return MemTypeOnCpuExplicitly(InputMemoryType(input_index)); }
+
+ bool IsOutputOnCpu(size_t output_index) const { return MemTypeOnCpuExplicitly(OutputMemoryType(output_index)); }
+
+ bool AllocateInputsContiguously() const { return allocate_inputs_contiguously_; }
+
+ OrtMemType OutputMemoryType(size_t output_index) const {
+ auto it = output_memory_type_args_.find(output_index);
+ if (it == output_memory_type_args_.end())
+ return default_outputs_mem_type_;
+ return it->second;
+ }
+
+ int ExecQueueId() const {
+ return exec_queue_id_;
+ }
+
+ bool IsConflict(const KernelDef& other) const;
+
+ uint64_t GetHash() const noexcept {
+ // if we need to support different hash versions we can update CalculateHash to take a version number
+ // and calculate any non-default versions dynamically. we only use this during kernel lookup so
+ // it's not performance critical
+ return hash_;
+ }
+
+ private:
+ friend class KernelDefBuilder;
+
+ // called once by KernelDefBuilder::Build
+ void CalculateHash();
+
+ // The operator name supported by <*this> kernel..
+ std::string op_name_;
+
+ // The operator since_version range supported by <*this> kernel.
+ // A kernel could support an operator definition between <op_since_version_start>
+ // and <op_since_version_end> (inclusive).
+ int op_since_version_start_ = 1;
+ int op_since_version_end_ = INT_MAX;
+
+ // The operator domain supported by <*this> kernel.
+ // Default to 'onnxruntime::kOnnxDomain'.
+ // Please note the behavior of std::string("") and std::string() are different
+ std::string op_domain_;
+
+ // The type of the execution provider.
+ std::string provider_type_;
+
+ // The supported data types for inputs/outputs.
+ // Key is input/output name defined in op schema, Value are supported types.
+ // note: std::map as we need the order to be deterministic for the hash
+ std::map<std::string, std::vector<MLDataType>> type_constraints_;
+
+ // An element <i, j> means that output j reuses the memory of input i.
+ std::vector<std::pair<int, int>> inplace_map_;
+
+ // An element <i, j> means that output j is an alias of input i.
+ std::vector<std::pair<int, int>> alias_map_;
+
+ // This variable stores <input_offset, output_offset> for the variadic alias mapping
+ // output 'i + output_offset' is an alias of input 'i + input_offset' for all i >= 0
+ optional<std::pair<int, int>> variadic_alias_offsets_;
+
+ // Require input tensors to be allocated contiguously.
+ bool allocate_inputs_contiguously_ = false;
+
+ // The memory types of inputs/outputs of this kernel
+ MemTypeMap input_memory_type_args_;
+ MemTypeMap output_memory_type_args_;
+
+ // execution command queue id, 0 for default queue in execution provider
+ int exec_queue_id_ = 0;
+ // Default memory type for all inputs
+ OrtMemType default_inputs_mem_type_{OrtMemTypeDefault};
+ // Default memory type for all outputs
+ OrtMemType default_outputs_mem_type_{OrtMemTypeDefault};
+
+ // hash of kernel definition for lookup in minimal build
+ uint64_t hash_ = 0;
+};
+
+class KernelDefBuilder {
+ public:
+ explicit KernelDefBuilder()
+ : kernel_def_(new KernelDef()) {}
+
+ KernelDefBuilder& SetName(const std::string& op_name);
+ KernelDefBuilder& SetName(const char* op_name);
+
+ KernelDefBuilder& SetDomain(const std::string& domain);
+ KernelDefBuilder& SetDomain(const char* domain);
+
+ /**
+ This kernel supports operator definition since <since_version> (to latest).
+ */
+ KernelDefBuilder& SinceVersion(int since_version) {
+ kernel_def_->op_since_version_start_ = since_version;
+ return *this;
+ }
+
+ /**
+ The start and end version should be set accordingly per version range for
+ each domain registered in OpSchemaRegistry::DomainToVersionRange in
+ \onnxruntime\onnxruntime\core\graph\op.h as below.
+ Key: domain. Value: <lowest version, highest version> pair.
+ std::unordered_map<std::string, std::pair<int, int>> map_;
+ */
+ KernelDefBuilder& SinceVersion(int since_version_start, int since_version_end) {
+ kernel_def_->op_since_version_start_ = since_version_start;
+ kernel_def_->op_since_version_end_ = since_version_end;
+ return *this;
+ }
+
+ /**
+ The execution provider type of the kernel.
+ */
+ KernelDefBuilder& Provider(onnxruntime::ProviderType provider_type);
+ KernelDefBuilder& Provider(const char* provider_type);
+
+ /**
+ Specify the set of types that this kernel supports. A further restriction
+ of the set of types specified in the op schema.
+ The arg name could be either op formal parameter name, say "X", or type
+ argument name specified in op schema, say "T".
+ */
+ KernelDefBuilder& TypeConstraint(const std::string& arg_name,
+ const std::vector<MLDataType>& supported_types);
+ KernelDefBuilder& TypeConstraint(const char* arg_name,
+ const std::vector<MLDataType>& supported_types);
+
+ /**
+ Like TypeConstraint but supports just a single type.
+ */
+ KernelDefBuilder& TypeConstraint(const std::string& arg_name, MLDataType supported_type);
+ KernelDefBuilder& TypeConstraint(const char* arg_name, MLDataType supported_type);
+
+ /**
+ Inplace mapping from inputs to outputs allowed.
+ It means that uplayer runtime could do memory in-place optimization
+ as it will not impact the correctness of this kernel.
+ */
+ KernelDefBuilder& MayInplace(const std::vector<std::pair<int, int>>& inplaces);
+ KernelDefBuilder& MayInplace(int input_index, int output_index);
+
+ /**
+ Alias mapping from inputs to outputs. Different from Inplace that the
+ content of the tensor is not changed. This is to take care of operators
+ such as Identity and Reshape.
+ */
+ KernelDefBuilder& Alias(const std::vector<std::pair<int, int>>& aliases);
+ KernelDefBuilder& Alias(int input_index, int output_index);
+
+ /**
+ Apply variadic number of alias mapping from inputs to outputs.
+ This is effectively applying Alias(i + input_offset, i + output_offset) for i >= 0
+ */
+ KernelDefBuilder& VariadicAlias(int input_offset, int output_offset);
+
+ /**
+ Specify that this kernel requires input tensors to be allocated
+ contiguously. This allows kernels to execute as a single large
+ computation, rather than numerous smaller computations.
+ */
+ KernelDefBuilder& AllocateInputsContiguously() {
+ kernel_def_->allocate_inputs_contiguously_ = true;
+ return *this;
+ }
+
+ /**
+ Specify that this kernel requires an input arg
+ in certain memory type (instead of the default, device memory).
+ */
+ template <OrtMemType T>
+ KernelDefBuilder& InputMemoryType(int input_index) {
+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T));
+ return *this;
+ }
+
+ KernelDefBuilder& InputMemoryType(OrtMemType type, int input_index) {
+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type));
+ return *this;
+ }
+
+ /**
+ Specify that this kernel requires input arguments
+ in certain memory type (instead of the default, device memory).
+ */
+ template <OrtMemType T>
+ KernelDefBuilder& InputMemoryType(const std::vector<int>& input_indexes) {
+ for (auto input_index : input_indexes) {
+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T));
+ }
+ return *this;
+ }
+
+ /**
+ Specify that this kernel provides an output arg
+ in certain memory type (instead of the default, device memory).
+ */
+ template <OrtMemType T>
+ KernelDefBuilder& OutputMemoryType(int output_index) {
+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T));
+ return *this;
+ }
+
+ KernelDefBuilder& OutputMemoryType(OrtMemType type, int output_index) {
+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, type));
+ return *this;
+ }
+
+ /**
+ Specify that this kernel provides an output arguments
+ in certain memory type (instead of the default, device memory).
+ */
+ template <OrtMemType T>
+ KernelDefBuilder& OutputMemoryType(const std::vector<int>& output_indexes) {
+ for (auto output_index : output_indexes) {
+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T));
+ }
+ return *this;
+ }
+
+ /**
+ Specify that this kernel runs on which execution queue in the provider
+ */
+ KernelDefBuilder& ExecQueueId(int queue_id) {
+ kernel_def_->exec_queue_id_ = queue_id;
+ return *this;
+ }
+
+ /**
+ Specify the default inputs memory type, if not specified, it is DefaultMemory
+ */
+ KernelDefBuilder& SetDefaultInputsMemoryType(OrtMemType mem_type) {
+ kernel_def_->default_inputs_mem_type_ = mem_type;
+ return *this;
+ }
+
+ /**
+ Specify the default outputs memory type, if not specified, it is DefaultMemory
+ */
+ KernelDefBuilder& SetDefaultOutputMemoryType(OrtMemType mem_type) {
+ kernel_def_->default_outputs_mem_type_ = mem_type;
+ return *this;
+ }
+
+ /**
+ Return the kernel definition, passing ownership of the KernelDef to the caller
+ */
+ std::unique_ptr<KernelDef> Build() {
+ kernel_def_->CalculateHash();
+ return std::move(kernel_def_);
+ }
+
+ private:
+ // we own the KernelDef until Build() is called.
+ std::unique_ptr<KernelDef> kernel_def_;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/kernel_registry.h b/include/onnxruntime/core/framework/kernel_registry.h
new file mode 100644
index 0000000..0542183
--- /dev/null
+++ b/include/onnxruntime/core/framework/kernel_registry.h
@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/framework/op_kernel.h"
+
+namespace onnxruntime {
+/**
+ * Each provider has a KernelRegistry. Often, the KernelRegistry only belongs to that specific provider.
+ *
+ */
+class KernelRegistry {
+ public:
+ KernelRegistry() = default;
+
+ // Register a kernel with kernel definition and function to create the kernel.
+ Status Register(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator) ORT_MUST_USE_RESULT;
+
+ Status Register(KernelCreateInfo&& create_info) ORT_MUST_USE_RESULT;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ static bool HasImplementationOf(const KernelRegistry& r, const onnxruntime::Node& node,
+ onnxruntime::ProviderType exec_provider) {
+ const KernelCreateInfo* info;
+ Status st = r.TryFindKernel(node, exec_provider, &info);
+ return st.IsOK();
+ }
+
+ // factory functions should always return a unique_ptr for maximum flexibility
+ // for its clients unless the factory is managing the lifecycle of the pointer
+ // itself.
+ // TODO(Task:132) Make usage of unique_ptr/shared_ptr as out param consistent
+ Status TryCreateKernel(const onnxruntime::Node& node, const IExecutionProvider& execution_provider,
+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors,
+ const OrtValueNameIdxMap& mlvalue_name_idx_map, const FuncManager& funcs_mgr,
+ const DataTransferManager& data_transfer_mgr,
+ std::unique_ptr<OpKernel>& op_kernel) const ORT_MUST_USE_RESULT;
+
+ // Check if an execution provider can create kernel for a node and return the kernel if so
+ Status TryFindKernel(const onnxruntime::Node& node, onnxruntime::ProviderType exec_provider,
+ const KernelCreateInfo** out) const;
+
+#endif
+
+ // Check if an execution provider can create kernel for a node and return the kernel if so.
+ // Kernel matching is via kernel_def_hash.
+ Status TryFindKernel(const onnxruntime::Node& node, onnxruntime::ProviderType exec_provider,
+ uint64_t kernel_def_hash,
+ const KernelCreateInfo** out) const;
+
+ bool IsEmpty() const { return kernel_creator_fn_map_.empty(); }
+
+#ifdef onnxruntime_PYBIND_EXPORT_OPSCHEMA
+ // This is used by the opkernel doc generator to enlist all registered operators for a given provider's opkernel
+ const KernelCreateMap& GetKernelCreateMap() const {
+ return kernel_creator_fn_map_;
+ }
+#endif
+
+ private:
+#if !defined(ORT_MINIMAL_BUILD)
+ // Check whether the types of inputs/outputs of the given node match the extra
+ // type-constraints of the given kernel. This serves two purposes: first, to
+ // select the right kernel implementation based on the types of the arguments
+ // when we have multiple kernels, e.g., Clip<float> and Clip<int>; second, to
+ // accommodate (and check) mapping of ONNX (specification) type to the onnxruntime
+ // implementation type (e.g., if we want to implement ONNX's float16 as a regular
+ // float in onnxruntime). (The second, however, requires a globally uniform mapping.)
+ //
+ // Note that this is not intended for type-checking the node against the ONNX
+ // type specification of the corresponding op, which is done before this check.
+ //
+ // if this function is called before graph partition, then node.provider is not set.
+ // In this case, kernel_def.provider must equal to exec_provider
+ // otherwise, kernel_def.provider must equal to node.provider. exec_provider is ignored.
+ static bool VerifyKernelDef(const onnxruntime::Node& node,
+ const KernelDef& kernel_def,
+ std::string& error_str);
+#endif
+
+ static std::string GetMapKey(const std::string& op_name, const std::string& domain, const std::string& provider) {
+ std::string key(op_name);
+ key.append(1, ' ').append(domain.empty() ? kOnnxDomainAlias : domain).append(1, ' ').append(provider);
+ return key;
+ }
+
+ static std::string GetMapKey(const KernelDef& kernel_def) {
+ return GetMapKey(kernel_def.OpName(), kernel_def.Domain(), kernel_def.Provider());
+ }
+ // Kernel create function map from op name to kernel creation info.
+ // key is opname+domain_name+provider_name
+ KernelCreateMap kernel_creator_fn_map_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/ml_value.h b/include/onnxruntime/core/framework/ml_value.h
new file mode 100644
index 0000000..27ffa58
--- /dev/null
+++ b/include/onnxruntime/core/framework/ml_value.h
@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+#include "core/common/common.h"
+#include "core/common/exceptions.h"
+#include "core/framework/allocator.h"
+#include "core/framework/data_types.h"
+#include "core/framework/tensor.h"
+
+namespace onnxruntime {
+class SparseTensor;
+class TensorSeq;
+} // namespace onnxruntime
+
+/**
+ Represents both tensors and non-tensors.
+*/
+struct OrtValue {
+ public:
+ OrtValue() : data_(nullptr) {}
+ ~OrtValue() = default;
+
+ OrtValue(void* pData, onnxruntime::MLDataType type, onnxruntime::DeleteFunc deleter) {
+ Init(pData, type, deleter);
+ }
+
+ void Init(void* pData, onnxruntime::MLDataType type, onnxruntime::DeleteFunc deleter) {
+ data_.reset(pData, deleter);
+ type_ = type;
+ }
+
+ bool IsAllocated() const {
+ return data_ && type_;
+ }
+
+ template <typename T>
+ const T& Get() const {
+ ORT_ENFORCE(onnxruntime::DataTypeImpl::GetType<T>() == type_, onnxruntime::DataTypeImpl::GetType<T>(), " != ", type_);
+ return *static_cast<T*>(data_.get());
+ }
+
+ template <typename T>
+ T* GetMutable() {
+ ORT_ENFORCE(onnxruntime::DataTypeImpl::GetType<T>() == type_, onnxruntime::DataTypeImpl::GetType<T>(), " != ", type_);
+ return static_cast<T*>(data_.get());
+ }
+
+ bool IsTensor() const noexcept {
+ return (type_ != nullptr && type_->IsTensorType());
+ }
+
+ bool IsTensorSequence() const noexcept {
+ return (type_ != nullptr && type_->IsTensorSequenceType());
+ }
+
+ bool IsSparseTensor() const noexcept {
+ return (type_ != nullptr && type_->IsSparseTensorType());
+ }
+
+ onnxruntime::MLDataType Type() const {
+ return type_;
+ }
+
+ onnxruntime::Fence_t Fence() const {
+ return fence_.get();
+ }
+
+ void SetFence(onnxruntime::FencePtr fence) {
+ fence_ = fence;
+ }
+
+ void ShareFenceWith(OrtValue& v) {
+ fence_ = v.fence_;
+ }
+
+ private:
+ std::shared_ptr<void> data_;
+ onnxruntime::MLDataType type_{nullptr};
+ onnxruntime::FencePtr fence_;
+};
+
+template <>
+inline const onnxruntime::Tensor& OrtValue::Get<onnxruntime::Tensor>() const {
+ ORT_ENFORCE(IsTensor(), "Trying to get a Tensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return *static_cast<onnxruntime::Tensor*>(data_.get());
+}
+
+template <>
+inline onnxruntime::Tensor* OrtValue::GetMutable<onnxruntime::Tensor>() {
+ ORT_ENFORCE(IsTensor(), "Trying to get a Tensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return static_cast<onnxruntime::Tensor*>(data_.get());
+}
+
+template <>
+inline const onnxruntime::TensorSeq& OrtValue::Get<onnxruntime::TensorSeq>() const {
+ ORT_ENFORCE(IsTensorSequence(), "Trying to get a TensorSeq, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return *static_cast<onnxruntime::TensorSeq*>(data_.get());
+}
+
+template <>
+inline onnxruntime::TensorSeq* OrtValue::GetMutable<onnxruntime::TensorSeq>() {
+ ORT_ENFORCE(IsTensorSequence(), "Trying to get a TensorSeq, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return static_cast<onnxruntime::TensorSeq*>(data_.get());
+}
+
+template <>
+inline const onnxruntime::SparseTensor& OrtValue::Get<onnxruntime::SparseTensor>() const {
+ ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return *static_cast<onnxruntime::SparseTensor*>(data_.get());
+}
+
+template <>
+inline onnxruntime::SparseTensor* OrtValue::GetMutable<onnxruntime::SparseTensor>() {
+ ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));
+ return static_cast<onnxruntime::SparseTensor*>(data_.get());
+}
+
+//TODO: remove the following line
+#define MLValue OrtValue
diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h
new file mode 100644
index 0000000..2e3937a
--- /dev/null
+++ b/include/onnxruntime/core/framework/op_kernel.h
@@ -0,0 +1,484 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <functional>
+
+#include "core/common/exceptions.h"
+#include "core/common/logging/logging.h"
+#include "core/common/status.h"
+#include "core/framework/execution_provider.h"
+#include "core/framework/kernel_def_builder.h"
+#include "core/framework/ml_value.h"
+#include "core/framework/op_kernel_info.h"
+#include "core/framework/op_node_proto_helper.h"
+#include "core/framework/tensor.h"
+#include "core/framework/sparse_tensor.h"
+#include "core/graph/constants.h"
+#include "core/graph/graph_viewer.h"
+#include "core/graph/onnx_protobuf.h"
+#include "gsl/gsl"
+
+namespace onnxruntime {
+class IExecutionFrame;
+class OpKernelContext;
+class OpKernelWrapper;
+namespace concurrency {
+class ThreadPool;
+}
+
+class OpKernel {
+ public:
+ using DoneCallback = std::function<void()>;
+
+ explicit OpKernel(const OpKernelInfo& info) : op_kernel_info_(info) {}
+ virtual ~OpKernel() = default;
+
+ const onnxruntime::Node& Node() const {
+ return op_kernel_info_.node();
+ }
+
+ const onnxruntime::KernelDef& KernelDef() const {
+ return op_kernel_info_.GetKernelDef();
+ }
+
+ virtual Status Compute(_Inout_ OpKernelContext* context) const ORT_MUST_USE_RESULT = 0;
+
+ virtual Status ComputeAsync(_Inout_ OpKernelContext*, DoneCallback) const ORT_MUST_USE_RESULT {
+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
+ }
+
+ // Override this function to PrePack initialized constant tensor to the format as needed.
+ // For example, MatMul kernel can pack the input B if it is constant like code below.
+ // Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override {
+ // is_packed = false;
+ // if (input_idx == 1) {
+ // this.Pack(tensor, this.buffer_);
+ // is_packed = true;
+ // }
+ // return Status::OK();
+ // }
+ // Please refer to MatMulIntegerToFloatBase for a complete example
+ // @param tesnor: The initialized constant tensor
+ // @param input_idx: The input index of the tensor in this kernel
+ // @param is_packed: Set it to true if the kernel packed the tensor or to false
+ // The kernel is responsible keep the packed data and related metadata if is_packed is set to true
+ // And the original intialized constant tensor will be released and not accessible anymore in Compute function.
+ virtual Status PrePack(const Tensor& /*tensor*/, int /*input_idx*/, bool& is_packed) {
+ is_packed = false;
+ return Status::OK();
+ }
+
+ const OrtMemoryInfo& Allocator(int id, OrtMemType mem_type) const {
+ return op_kernel_info_.GetMemoryInfo(id, mem_type);
+ }
+
+ const OpKernelInfo& Info() const { return op_kernel_info_; }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OpKernel);
+ OpKernelInfo op_kernel_info_;
+};
+
+class OpKernelContext {
+ public:
+ using ArgMap = std::unordered_map<std::string, size_t>;
+
+ OpKernelContext(_Inout_ IExecutionFrame* frame, _In_ const OpKernel* kernel,
+ _In_opt_ concurrency::ThreadPool* threadpool, _In_ const logging::Logger& logger);
+
+ virtual ~OpKernelContext() = default;
+
+ /**
+ Return the number of inputs for a variadic argument.
+ @param arg_num The operator argument number.
+ @returns Number of inputs the argument has.
+ */
+ int NumVariadicInputs(size_t arg_num) const;
+
+ MLDataType InputType(int index) const;
+ MLDataType OutputType(int index) const;
+
+ template <typename T>
+ const T* Input(int index) const {
+ const OrtValue* p_ml_value = GetInputMLValue(index);
+ ORT_TRY {
+ return p_ml_value ? &(p_ml_value->Get<T>()) : nullptr;
+ }
+ ORT_CATCH(const std::exception& /*e*/) {
+ ORT_THROW("Missing Input: " + kernel_->Node().InputDefs()[index]->Name());
+ }
+ }
+
+ // Fetch a required input, enforcing that it is present.
+ template <typename T>
+ const T& RequiredInput(int index) const {
+ const T* input_ptr = Input<T>(index);
+ ORT_ENFORCE(input_ptr, "Required input at index ", index, " is not present.");
+ return *input_ptr;
+ }
+
+ // Fetch output (non-tensor) with specified index.
+ template <typename T>
+ T* Output(int index) {
+ if (index < 0 || index >= OutputCount())
+ return nullptr;
+
+ OrtValue* p_ml_value = GetOrCreateOutputMLValue(index);
+ return p_ml_value ? p_ml_value->GetMutable<T>() : nullptr;
+ }
+
+ // In the case that memory allocation has not been done for an output tensor,
+ // The memory allocation will be done on-the-fly with given tensor shape.
+ // Return nullptr if the output is an unused optional output.
+ Tensor* Output(int index, const TensorShape& shape);
+ Tensor* Output(int index, const std::vector<int64_t>& shape);
+ Tensor* Output(int index, const std::initializer_list<int64_t>& shape);
+
+ // Fetch a required tensor output, enforcing that it is present.
+ Tensor& RequiredOutput(int index, const TensorShape& shape) {
+ Tensor* output_ptr = Output(index, shape);
+ ORT_ENFORCE(output_ptr, "Required output at index ", index, " is not present.");
+ return *output_ptr;
+ }
+
+ // Fetch a sparse-tensor output corresponding to the specified index.
+ // num_values must specify the number of non-zero values (commonly known as NNZ/nnz),
+ // and shape must specify the shape of the underlying dense-tensor.
+ // Memory allocation for the output may happen when this method is invoked,
+ // unless static optimization pre-allocates it.
+ SparseTensor* Output(int index, size_t num_values, const TensorShape& shape);
+
+ // Retrieve indexed shape obtained from memory planning before actual
+ // computation. If the indexed shape cannot be inferred, this function returns
+ // false.
+ bool TryGetInferredInputShape(int index, TensorShape& shape) const;
+
+ // Retrieve indexed shape obtained from memory planning before actual
+ // computation. If the indexed shape cannot be inferred, this function returns
+ // false.
+ bool TryGetInferredOutputShape(int index, TensorShape& shape) const;
+
+ const logging::Logger& Logger() const {
+ return *logger_;
+ }
+
+ // always >= 0
+ int InputCount() const {
+ return static_cast<int>(kernel_->Node().InputDefs().size());
+ }
+
+ // always >= 0
+ int ImplicitInputCount() const {
+ return static_cast<int>(kernel_->Node().ImplicitInputDefs().size());
+ }
+
+ // always >= 0
+ int OutputCount() const {
+ return static_cast<int>(kernel_->Node().OutputDefs().size());
+ }
+
+ /**
+ Return an allocator on device 0, with memtype of OrtMemTypeDefault.
+ @remarks Use SafeInt when calculating the size of memory to allocate using AllocatorPtr->Alloc.
+ */
+ Status GetTempSpaceAllocator(AllocatorPtr* output) const ORT_MUST_USE_RESULT;
+
+ /**
+ Return the fence of current node's input.
+ @param index The index of the input.
+ @returns Point to the Fence of the input OrtValue.
+ It is null if the input OrtValue doesn't have fence or the input is optional.
+ */
+ Fence_t InputFence(int index) const;
+
+ /**
+ Return the fence of current node's implicit input.
+ @param index The index of the implicit input.
+ @returns Point to the Fence of the implicit input OrtValue.
+ It is null if the input OrtValue doesn't have fence or the input is optional.
+ */
+ Fence_t ImplicitInputFence(int index) const;
+
+ /**
+ Return the fence of current node's output identifed by index.
+ @param index The index of the output.
+ @returns Point to the Fence of the output OrtValue.
+ It is null if the output OrtValue doesn't have fence or the output is optional.
+ */
+ Fence_t OutputFence(int index) const;
+
+ /**
+ Return the device id that current kernel runs on.
+ */
+ int GetDeviceId() const {
+ return kernel_->Info().GetExecutionProvider()->GetDeviceId();
+ }
+
+ /**
+ Returns the opset domain of the underlying kernel
+ **/
+ const std::string& GetOpDomain() const;
+
+ /**
+ Returns the optype of the underlying kernel
+ **/
+ const std::string& GetOpType() const;
+
+ /**
+ Returns the node name of the underlying kernel
+ **/
+ const std::string& GetNodeName() const;
+
+ /**
+ Returns the intra-op threadpool, if available.
+ */
+ _Ret_maybenull_ onnxruntime::concurrency::ThreadPool* GetOperatorThreadPool() const { return threadpool_; }
+
+ /**
+ Returns whether deterministic computation is preferred.
+ */
+ virtual bool GetUseDeterministicCompute() const {
+ return true;
+ }
+
+ protected:
+ onnxruntime::NodeIndex GetNodeIndex() const;
+
+ const OrtValue* GetInputMLValue(int index) const;
+ const OrtValue* GetImplicitInputMLValue(int index) const;
+ OrtValue* GetOutputMLValue(int index);
+
+ // Creates the OrtValue* based on the shape, if it does not exist
+ // The parameter nnz is used only for sparse-tensors and indicates the
+ // number of non-zero values (the number of elements in the values buffer allocated).
+ OrtValue* OutputMLValue(int index, const TensorShape& shape, size_t nnz = 0);
+
+ private:
+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(OpKernelContext);
+
+ OrtValue* GetOrCreateOutputMLValue(int index);
+
+ int GetInputArgIndex(int index) const;
+ int GetImplicitInputArgIndex(int index) const;
+ int GetOutputArgIndex(int index) const;
+
+ IExecutionFrame* const execution_frame_;
+ const OpKernel* const kernel_;
+ concurrency::ThreadPool* const threadpool_;
+ const logging::Logger* const logger_;
+
+ // The argument starting index in ExecutionFrame.
+ int node_input_start_index_{-1};
+ int node_implicit_input_start_index_{-1};
+ int node_output_start_index_{-1};
+};
+
+// Fetching output tensor without shape is not allowed except when it already exists
+template <>
+inline Tensor* OpKernelContext::Output<Tensor>(int index) {
+ OrtValue* p_ml_value = GetOutputMLValue(index);
+ ORT_ENFORCE(p_ml_value, "Please fetch output tensor with specified shape.");
+ return p_ml_value->GetMutable<Tensor>();
+}
+
+using KernelCreateFn = std::function<OpKernel*(const OpKernelInfo& info)>;
+using KernelCreatePtrFn = std::add_pointer<OpKernel*(const OpKernelInfo& info)>::type;
+
+struct KernelCreateInfo {
+ std::unique_ptr<KernelDef> kernel_def; // Owned and stored in the global kernel registry.
+ KernelCreateFn kernel_create_func;
+ Status status;
+
+ KernelCreateInfo(std::unique_ptr<KernelDef> definition,
+ KernelCreateFn create_func)
+ : kernel_def(std::move(definition)),
+ kernel_create_func(create_func) {}
+
+ KernelCreateInfo(KernelCreateInfo&& other) noexcept
+ : kernel_def(std::move(other.kernel_def)),
+ kernel_create_func(std::move(other.kernel_create_func)) {}
+
+ KernelCreateInfo() = default;
+};
+
+using KernelCreateMap = std::multimap<std::string, KernelCreateInfo>;
+
+// Forward declarations for the non-specialized BuildKernelCreateInfo method.
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+
+namespace ml {
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+} // namespace ml
+
+namespace contrib {
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+} // namespace contrib
+
+namespace featurizers {
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+} // namespace featurizers
+
+namespace contrib {
+namespace cuda {
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+} // namespace cuda
+} // namespace contrib
+
+namespace contrib {
+namespace rocm {
+template <typename T>
+KernelCreateInfo BuildKernelCreateInfo();
+} // namespace rocm
+} // namespace contrib
+
+using BuildKernelCreateInfoFn = KernelCreateInfo (*)();
+
+// Naming convention for operator kernel classes
+#define ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name) \
+ provider##_##name##_##domain##_ver##ver
+
+#define ONNX_CPU_OPERATOR_KERNEL(name, ver, builder, ...) \
+ ONNX_OPERATOR_KERNEL_EX(name, kOnnxDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_ML_KERNEL(name, ver, builder, ...) \
+ ONNX_OPERATOR_KERNEL_EX(name, kMLDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_OPERATOR_KERNEL_EX(name, domain, ver, provider, builder, ...) \
+ class ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(ver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+#define ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name) \
+ provider##_##name##_##domain##_ver##startver##_##endver
+
+#define ONNX_CPU_OPERATOR_VERSIONED_KERNEL(name, startver, endver, builder, ...) \
+ ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kOnnxDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_VERSIONED_ML_KERNEL(name, startver, endver, builder, ...) \
+ ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kMLDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, domain, startver, endver, provider, builder, ...) \
+ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(startver, endver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+#define ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name) \
+ provider##_##name##_##domain##_ver##ver##_##type
+
+#define ONNX_CPU_OPERATOR_TYPED_KERNEL(name, ver, type, builder, ...) \
+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kOnnxDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_TYPED_ML_KERNEL(name, ver, type, builder, ...) \
+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMLDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_TYPED_MS_KERNEL(name, ver, type, builder, ...) \
+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMSDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_OPERATOR_TYPED_KERNEL_EX(name, domain, ver, type, provider, builder, ...) \
+ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(ver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+#define ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name) \
+ provider##_##name##_##domain##_ver##ver##_##type1##_##type2
+
+#define ONNX_OPERATOR_TWO_TYPED_KERNEL_EX(name, domain, ver, type1, type2, provider, builder, ...) \
+ class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(ver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name) \
+ provider##_##name##_##domain##_ver##startver##_##endver##_##type
+
+#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(name, startver, endver, type, builder, ...) \
+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kOnnxDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_ML_KERNEL(name, startver, endver, type, builder, ...) \
+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMLDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_MS_KERNEL(name, startver, endver, type, builder, ...) \
+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMSDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)
+
+#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, domain, startver, endver, type, provider, builder, ...) \
+ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(startver, endver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+#define ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name) \
+ provider##_##name##_##domain##_ver##startver##_##endver##_##type1##_##type2
+
+#define ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_EX(name, domain, startver, endver, type1, type2, provider, builder, ...) \
+ class ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name); \
+ template <> \
+ KernelCreateInfo \
+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name)>() { \
+ return KernelCreateInfo( \
+ builder.SetName(#name) \
+ .SetDomain(domain) \
+ .SinceVersion(startver, endver) \
+ .Provider(provider) \
+ .Build(), \
+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \
+ }
+
+// Use within macro definitions to create a custom vector of constraints.
+// Example: #define REG_KERNEL(OP, VERSION, KERNEL_CLASS, Type, ...)
+// .TypeConstraint("T", BuildKernelDefConstraints<Type, __VA_ARGS_>())
+template <typename T, typename... Types>
+inline std::vector<MLDataType> BuildKernelDefConstraints() {
+ return {DataTypeImpl::GetTensorType<T>(), DataTypeImpl::GetTensorType<Types>()...};
+}
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/op_kernel_info.h b/include/onnxruntime/core/framework/op_kernel_info.h
new file mode 100644
index 0000000..40b8c9a
--- /dev/null
+++ b/include/onnxruntime/core/framework/op_kernel_info.h
@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/framework/execution_provider.h"
+#include "core/framework/kernel_def_builder.h"
+#include "core/framework/ml_value.h"
+#include "core/framework/op_node_proto_helper.h"
+#include "core/graph/graph_viewer.h"
+#include "gsl/gsl"
+
+namespace onnxruntime {
+
+class OrtValueNameIdxMap;
+class FuncManager;
+class DataTransferManager;
+
+// A very light-weight class, which works as an aggregated
+// view of all data needed for constructing a Kernel instance.
+// NOTE: it does not own/hold any objects.
+class OpKernelInfo : public OpNodeProtoHelper<ProtoHelperNodeContext> {
+ public:
+ explicit OpKernelInfo(const onnxruntime::Node& node,
+ const KernelDef& kernel_def,
+ const IExecutionProvider& execution_provider,
+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors,
+ const OrtValueNameIdxMap& mlvalue_name_idx_map,
+ const FuncManager& funcs_mgr,
+ const DataTransferManager& data_transfer_mgr);
+
+ OpKernelInfo(const OpKernelInfo& other);
+
+ const OrtMemoryInfo& GetMemoryInfo(int device_id, OrtMemType mem_type) const;
+
+ AllocatorPtr GetAllocator(int device_id, OrtMemType mem_type) const;
+
+ const KernelDef& GetKernelDef() const;
+
+ const IExecutionProvider* GetExecutionProvider() const noexcept;
+
+ const DataTransferManager& GetDataTransferManager() const noexcept;
+
+ const onnxruntime::Node& node() const noexcept;
+
+ bool TryGetConstantInput(int input_index, const Tensor** constant_input_value) const;
+
+ common::Status GetFusedFuncs(NodeComputeInfo*& compute_info) const;
+
+ private:
+ ORT_DISALLOW_MOVE(OpKernelInfo);
+ ORT_DISALLOW_ASSIGNMENT(OpKernelInfo);
+
+ const onnxruntime::Node& node_;
+ const KernelDef& kernel_def_;
+ // For non cpu/cuda case, this pointer should be set so that function kernel
+ // will delegate kernel compute call to <execution_provider> compute call.
+ gsl::not_null<const ::onnxruntime::IExecutionProvider*> execution_provider_;
+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors_;
+ const OrtValueNameIdxMap& ort_value_name_idx_map_;
+ const FuncManager& funcs_mgr_;
+ const DataTransferManager& data_transfer_mgr_;
+ ProtoHelperNodeContext proto_helper_context_;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/op_node_proto_helper.h b/include/onnxruntime/core/framework/op_node_proto_helper.h
new file mode 100644
index 0000000..7443459
--- /dev/null
+++ b/include/onnxruntime/core/framework/op_node_proto_helper.h
@@ -0,0 +1,157 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/status.h"
+#include "core/graph/graph_viewer.h"
+#include "gsl/gsl"
+
+#ifdef __has_attribute
+#define ORT_HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+#define ORT_HAVE_ATTRIBUTE(x) 0
+#endif
+
+#if ORT_HAVE_ATTRIBUTE(nodiscard)
+#define MUST_USE_RESULT [[nodiscard]]
+#elif defined(__clang__) && ORT_HAVE_ATTRIBUTE(warn_unused_result)
+#define MUST_USE_RESULT __attribute__((warn_unused_result))
+#else
+#define MUST_USE_RESULT
+#endif
+
+class IMLOpKernel;
+
+namespace onnxruntime {
+
+/**
+ A set of wrappers with common signatures for use with both OpKernelInfo
+ (as its base class) and InferenceContext. Used by ABI kernels for both
+ shape / type inference and kernel construction
+*/
+template <class Impl_t>
+class OpNodeProtoHelper {
+ public:
+ explicit OpNodeProtoHelper(const Impl_t* impl) : impl_(impl) {}
+
+ /**
+ Get a single attribute
+ Call this function for a required attribute or when a default value for an optional attribute is specified in the op schema
+ */
+ template <typename T>
+ MUST_USE_RESULT Status GetAttr(const std::string& name, T* value) const;
+
+ /**
+ Get a single attribute
+ Call this function only when a default value for an optional attribute isn't specified in the op schema
+ */
+ template <typename T>
+ T GetAttrOrDefault(const std::string& name, const T& default_value) const {
+ T tmp;
+ return GetAttr<T>(name, &tmp).IsOK() ? tmp : default_value;
+ }
+
+ /**
+ Get a single attribute
+ Call this function only when a default value for an optional attribute isn't specified in the op schema
+ */
+ template <typename T>
+ void GetAttrOrDefault(const std::string& name, T* value, const T& default_value) const {
+ if (!GetAttr<T>(name, value).IsOK())
+ *value = default_value;
+ }
+
+ /**
+ Get repeated attributes
+ Call this function only when a default value for an optional attribute isn't specified in the op schema
+ */
+ template <typename T>
+ MUST_USE_RESULT std::vector<T> GetAttrsOrDefault(const std::string& name, const std::vector<T>& default_value = std::vector<T>{}) const {
+ std::vector<T> tmp;
+ return GetAttrs<T>(name, tmp).IsOK() ? tmp : default_value;
+ }
+
+ /**
+ Get repeated attributes
+ */
+ template <typename T>
+ MUST_USE_RESULT Status GetAttrs(const std::string& name, std::vector<T>& values) const;
+
+ template <typename T>
+ MUST_USE_RESULT Status GetAttrs(const std::string& name, gsl::span<T> values) const;
+
+ /// <summary>
+ /// Return a gsl::span that points to an array of primitive types held by AttributeProto
+ /// This function allows to avoid copying big attributes locally into a kernel and operate on
+ /// AttributeProto data directly.
+ ///
+ /// Does not apply to strings, Tensors and Sparse Tensors that require special treatment.
+ /// </summary>
+ /// <typeparam name="T">Primitive type contained in the array</typeparam>
+ /// <param name="name">Attribute name</param>
+ /// <param name="values">Attribute data in a span, out parameter</param>
+ /// <returns>Status</returns>
+ template <typename T>
+ MUST_USE_RESULT Status GetAttrsAsSpan(const std::string& name, gsl::span<const T>& values) const;
+
+ MUST_USE_RESULT Status GetAttrsStringRefs(const std::string& name,
+ std::vector<std::reference_wrapper<const std::string>>& refs) const;
+
+ uint32_t GetPrimitiveAttrElementCount(ONNX_NAMESPACE::AttributeProto_AttributeType type,
+ const std::string& name) const noexcept;
+
+ bool HasPrimitiveAttribute(ONNX_NAMESPACE::AttributeProto_AttributeType type,
+ const std::string& name) const noexcept;
+
+ uint32_t GetInputCount() const {
+ return gsl::narrow_cast<uint32_t>(impl_->getNumInputs());
+ }
+
+ uint32_t GetOutputCount() const {
+ return gsl::narrow_cast<uint32_t>(impl_->getNumOutputs());
+ }
+
+ const ONNX_NAMESPACE::TypeProto* GetInputType(size_t index) const {
+ return impl_->getInputType(index);
+ }
+
+ const ONNX_NAMESPACE::TypeProto* GetOutputType(size_t index) const {
+ // Work around lack of a const method from the onnx InferenceContext interface
+ return const_cast<Impl_t*>(impl_)->getOutputType(index);
+ }
+
+ // Try to query an attribute, returning nullptr if it doesn't exist
+ const ONNX_NAMESPACE::AttributeProto* TryGetAttribute(const std::string& name) const {
+ return impl_->getAttribute(name);
+ }
+
+ const ONNX_NAMESPACE::AttributeProto* GetAttribute(const std::string& name) const {
+ const ONNX_NAMESPACE::AttributeProto* attr = TryGetAttribute(name);
+ ORT_ENFORCE(attr != nullptr);
+ return attr;
+ }
+
+ private:
+ OpNodeProtoHelper() = delete;
+ const Impl_t* impl_ = nullptr;
+};
+
+// The methods on the following class are called by OpNodeProtoHelper, implementing
+// the same signatures as InferenceContext other than const-ness.
+class ProtoHelperNodeContext {
+ public:
+ explicit ProtoHelperNodeContext(const onnxruntime::Node& node) : node_(node) {}
+ ProtoHelperNodeContext() = delete;
+
+ const ONNX_NAMESPACE::AttributeProto* getAttribute(const std::string& name) const;
+ size_t getNumInputs() const;
+ const ONNX_NAMESPACE::TypeProto* getInputType(size_t index) const;
+ size_t getNumOutputs() const;
+ const ONNX_NAMESPACE::TypeProto* getOutputType(size_t index) const;
+
+ private:
+ const onnxruntime::Node& node_;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/ortdevice.h b/include/onnxruntime/core/framework/ortdevice.h
new file mode 100644
index 0000000..afc1518
--- /dev/null
+++ b/include/onnxruntime/core/framework/ortdevice.h
@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <sstream>
+
+// Struct to represent a physical device.
+struct OrtDevice {
+ using DeviceType = int8_t;
+ using MemoryType = int8_t;
+ using DeviceId = int16_t;
+
+ // Pre-defined device types.
+ static const DeviceType CPU = 0;
+ static const DeviceType GPU = 1; // Nvidia or AMD
+ static const DeviceType FPGA = 2;
+
+ struct MemType {
+ // Pre-defined memory types.
+ static const MemoryType DEFAULT = 0;
+ static const MemoryType CUDA_PINNED = 1;
+ static const MemoryType HIP_PINNED = 2;
+ };
+
+ constexpr OrtDevice(DeviceType device_type_, MemoryType memory_type_, DeviceId device_id_)
+ : device_type(device_type_),
+ memory_type(memory_type_),
+ device_id(device_id_) {}
+
+ constexpr OrtDevice() : OrtDevice(CPU, MemType::DEFAULT, 0) {}
+
+ DeviceType Type() const {
+ return device_type;
+ }
+
+ MemoryType MemType() const {
+ return memory_type;
+ }
+
+ DeviceId Id() const {
+ return device_id;
+ }
+
+ std::string ToString() const {
+ std::ostringstream ostr;
+ ostr << "Device:["
+ << "DeviceType:" << static_cast<int>(device_type)
+ << " MemoryType:" << static_cast<int>(memory_type)
+ << " DeviceId:" << device_id
+ << "]";
+ return ostr.str();
+ }
+
+ private:
+ // Device type.
+ DeviceType device_type;
+
+ // Memory type.
+ MemoryType memory_type;
+
+ // Device index.
+ DeviceId device_id;
+};
+
+inline bool operator==(const OrtDevice& left, const OrtDevice& other) {
+ return left.Id() == other.Id() && left.MemType() == other.MemType() && left.Type() == other.Type();
+}
+
+inline bool operator!=(const OrtDevice& left, const OrtDevice& other) {
+ return !(left == other);
+}
diff --git a/include/onnxruntime/core/framework/ortmemoryinfo.h b/include/onnxruntime/core/framework/ortmemoryinfo.h
new file mode 100644
index 0000000..77df903
--- /dev/null
+++ b/include/onnxruntime/core/framework/ortmemoryinfo.h
@@ -0,0 +1,63 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+struct OrtMemoryInfo {
+ OrtMemoryInfo() = default; // to allow default construction of Tensor
+
+ // use string for name, so we could have customized allocator in execution provider.
+ const char* name = nullptr;
+ int id = -1;
+ OrtMemType mem_type = OrtMemTypeDefault;
+ OrtAllocatorType alloc_type = Invalid;
+ OrtDevice device;
+
+ constexpr OrtMemoryInfo(const char* name_, OrtAllocatorType type_, OrtDevice device_ = OrtDevice(), int id_ = 0,
+ OrtMemType mem_type_ = OrtMemTypeDefault)
+#if ((defined(__GNUC__) && __GNUC__ > 4) || defined(__clang__))
+ // this causes a spurious error in CentOS gcc 4.8 build so disable if GCC version < 5
+ __attribute__((nonnull))
+#endif
+ : name(name_),
+ id(id_),
+ mem_type(mem_type_),
+ alloc_type(type_),
+ device(device_) {
+ }
+
+ // To make OrtMemoryInfo become a valid key in std map
+ bool operator<(const OrtMemoryInfo& other) const {
+ if (alloc_type != other.alloc_type)
+ return alloc_type < other.alloc_type;
+ if (mem_type != other.mem_type)
+ return mem_type < other.mem_type;
+ if (id != other.id)
+ return id < other.id;
+
+ return strcmp(name, other.name) < 0;
+ }
+
+ std::string ToString() const {
+ std::ostringstream ostr;
+ ostr << "OrtMemoryInfo:["
+ << "name:" << name
+ << " id:" << id
+ << " OrtMemType:" << mem_type
+ << " OrtAllocatorType:" << alloc_type
+ << " " << device.ToString()
+ << "]";
+ return ostr.str();
+ }
+};
+
+inline bool operator==(const OrtMemoryInfo& left, const OrtMemoryInfo& other) {
+ return left.mem_type == other.mem_type &&
+ left.alloc_type == other.alloc_type &&
+ left.id == other.id &&
+ strcmp(left.name, other.name) == 0;
+}
+
+inline bool operator!=(const OrtMemoryInfo& lhs, const OrtMemoryInfo& rhs) { return !(lhs == rhs); }
+
+std::ostream& operator<<(std::ostream& out, const OrtMemoryInfo& info);
diff --git a/include/onnxruntime/core/framework/provider_options.h b/include/onnxruntime/core/framework/provider_options.h
new file mode 100644
index 0000000..aab13e8
--- /dev/null
+++ b/include/onnxruntime/core/framework/provider_options.h
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace onnxruntime {
+
+// data types for execution provider options
+
+using ProviderOptions = std::unordered_map<std::string, std::string>;
+using ProviderOptionsVector = std::vector<ProviderOptions>;
+using ProviderOptionsMap = std::unordered_map<std::string, ProviderOptions>;
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/provider_options_utils.h b/include/onnxruntime/core/framework/provider_options_utils.h
new file mode 100644
index 0000000..e326481
--- /dev/null
+++ b/include/onnxruntime/core/framework/provider_options_utils.h
@@ -0,0 +1,165 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <algorithm>
+#include <functional>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+
+#include "core/common/common.h"
+#include "core/common/parse_string.h"
+#include "core/framework/provider_options.h"
+
+namespace onnxruntime {
+
+template <typename TEnum>
+using EnumNameMapping = std::vector<std::pair<TEnum, std::string>>;
+
+/**
+ * Given a mapping and an enumeration value, gets the corresponding name.
+ */
+template <typename TEnum>
+Status EnumToName(const EnumNameMapping<TEnum>& mapping, TEnum value, std::string& name) {
+ const auto it = std::find_if(
+ mapping.begin(), mapping.end(),
+ [&value](const std::pair<TEnum, std::string>& entry) {
+ return entry.first == value;
+ });
+ ORT_RETURN_IF(
+ it == mapping.end(),
+ "Failed to map enum value to name: ", static_cast<typename std::underlying_type<TEnum>::type>(value));
+ name = it->second;
+ return Status::OK();
+}
+
+template <typename TEnum>
+std::string EnumToName(const EnumNameMapping<TEnum>& mapping, TEnum value) {
+ std::string name;
+ ORT_THROW_IF_ERROR(EnumToName(mapping, value, name));
+ return name;
+}
+
+/**
+ * Given a mapping and a name, gets the corresponding enumeration value.
+ */
+template <typename TEnum>
+Status NameToEnum(
+ const EnumNameMapping<TEnum>& mapping, const std::string& name, TEnum& value) {
+ const auto it = std::find_if(
+ mapping.begin(), mapping.end(),
+ [&name](const std::pair<TEnum, std::string>& entry) {
+ return entry.second == name;
+ });
+ ORT_RETURN_IF(
+ it == mapping.end(),
+ "Failed to map enum name to value: ", name);
+ value = it->first;
+ return Status::OK();
+}
+
+template <typename TEnum>
+TEnum NameToEnum(const EnumNameMapping<TEnum>& mapping, const std::string& name) {
+ TEnum value;
+ ORT_THROW_IF_ERROR(NameToEnum(mapping, name, value));
+ return value;
+}
+
+class ProviderOptionsParser {
+ public:
+ /**
+ * Adds a parser for a particular provider option value.
+ *
+ * @param name The provider option name.
+ * @param value_parser An object that parses the option value.
+ * It should be callable with the following signature and return
+ * whether the parsing was successful:
+ * Status value_parser(const std::string&)
+ *
+ * @return The current ProviderOptionsParser instance.
+ */
+ template <typename ValueParserType>
+ ProviderOptionsParser& AddValueParser(
+ const std::string& name, ValueParserType value_parser) {
+ ORT_ENFORCE(
+ value_parsers_.emplace(name, ValueParser{value_parser}).second,
+ "Provider option \"", name, "\" already has a value parser.");
+ return *this;
+ }
+
+ /**
+ * Adds a parser for a particular provider option value which converts a
+ * value to the right type and assigns it to the given reference.
+ *
+ * IMPORTANT: This function stores a reference to the destination variable.
+ * The caller must ensure that the reference is valid when Parse() is called!
+ *
+ * @param name The provider option name.
+ * @param dest The destination variable reference.
+ *
+ * @return The current ProviderOptionsParser instance.
+ */
+ template <typename ValueType>
+ ProviderOptionsParser& AddAssignmentToReference(
+ const std::string& name, ValueType& dest) {
+ return AddValueParser(
+ name,
+ [&dest](const std::string& value_str) -> Status {
+ return ParseStringWithClassicLocale(value_str, dest);
+ });
+ }
+
+ /**
+ * Adds a parser for a particular provider option value which maps an
+ * enumeration name to a value and assigns it to the given reference.
+ *
+ * IMPORTANT: This function stores references to the mapping and destination
+ * variables. The caller must ensure that the references are valid when
+ * Parse() is called!
+ *
+ * @param name The provider option name.
+ * @param mapping The enumeration value to name mapping.
+ * @param dest The destination variable reference.
+ *
+ * @return The current ProviderOptionsParser instance.
+ */
+ template <typename EnumType>
+ ProviderOptionsParser& AddAssignmentToEnumReference(
+ const std::string& name, const EnumNameMapping<EnumType>& mapping, EnumType& dest) {
+ return AddValueParser(
+ name,
+ [&mapping, &dest](const std::string& value_str) -> Status {
+ return NameToEnum(mapping, value_str, dest);
+ });
+ }
+
+ /**
+ * Parses the given provider options.
+ */
+ Status Parse(const ProviderOptions& options) const {
+ for (const auto& option : options) {
+ const auto& name = option.first;
+ const auto& value_str = option.second;
+
+ const auto value_parser_it = value_parsers_.find(name);
+ ORT_RETURN_IF(
+ value_parser_it == value_parsers_.end(),
+ "Unknown provider option: \"", name, "\".");
+
+ const auto parse_status = value_parser_it->second(value_str);
+ ORT_RETURN_IF_NOT(
+ parse_status.IsOK(),
+ "Failed to parse provider option \"", name, "\": ", parse_status.ErrorMessage());
+ }
+
+ return Status::OK();
+ }
+
+ private:
+ using ValueParser = std::function<Status(const std::string&)>;
+ std::unordered_map<std::string, ValueParser> value_parsers_;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/provider_shutdown.h b/include/onnxruntime/core/framework/provider_shutdown.h
new file mode 100644
index 0000000..7488f12
--- /dev/null
+++ b/include/onnxruntime/core/framework/provider_shutdown.h
@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+void UnloadSharedProviders();
+}
diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.h
new file mode 100644
index 0000000..85f6e2f
--- /dev/null
+++ b/include/onnxruntime/core/framework/run_options.h
@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <string>
+#include <atomic>
+#include "core/session/onnxruntime_c_api.h"
+
+/**
+ * Configuration information for a Run call.
+ */
+struct OrtRunOptions {
+ /// Log severity. See https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h
+ /// Default = -1 (use the log severity from the InferenceSession that the Run is for).
+ int run_log_severity_level = -1;
+ int run_log_verbosity_level = 0; ///< VLOG level if debug build and run_log_severity_level is 0 (VERBOSE).
+ std::string run_tag; ///< A tag for the Run() calls using this.
+
+ // Set to 'true' to ensure the termination of all the outstanding Run() calls
+ // that use this OrtRunOptions instance. Some of the outstanding Run() calls may
+ // be forced to terminate with an error status.
+ bool terminate = false;
+
+ // Set to 'true' to run only the nodes from feeds to required fetches.
+ // So it is possible that only some of the nodes are executed.
+ bool only_execute_path_to_fetches = false;
+
+#ifdef ENABLE_TRAINING
+ // Set to 'true' to run in training mode.
+ bool training_mode = true;
+#endif
+
+ OrtRunOptions() = default;
+ ~OrtRunOptions() = default;
+};
+
+namespace onnxruntime {
+using RunOptions = OrtRunOptions;
+}
diff --git a/include/onnxruntime/core/framework/sparse_tensor.h b/include/onnxruntime/core/framework/sparse_tensor.h
new file mode 100644
index 0000000..ec0abb9
--- /dev/null
+++ b/include/onnxruntime/core/framework/sparse_tensor.h
@@ -0,0 +1,72 @@
+#pragma once
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/framework/data_types.h"
+#include "core/framework/tensor_shape.h"
+#include "core/framework/tensor.h"
+
+namespace onnxruntime {
+
+/**
+ * @brief This class implements SparseTensor.
+ * We represent a SparseTensor as a triple <values, indices, shape>. "values" and "indices" themselves
+ * are implemented as Tensors.
+ * We follow the Tensor design for memory ownership/management: a sparse-tensor does not own the "value"
+ * or "indices" tensors.
+ */
+
+class SparseTensor final {
+ public:
+ SparseTensor(MLDataType elt_type,
+ const TensorShape& shape,
+ size_t nnz,
+ void* values_data,
+ void* indices_data,
+ const OrtMemoryInfo& memory_info);
+
+ SparseTensor(MLDataType elt_type,
+ const TensorShape& shape,
+ size_t nnz,
+ std::shared_ptr<IAllocator> allocator);
+
+ ~SparseTensor() = default;
+
+ // For now, disallow all copy, assignment, and move.
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SparseTensor);
+
+ // Returns the number of entries in the values tensor (aka "NNZ" or "number of nonzero values")
+ size_t NumValues() const { return static_cast<size_t>(values_.Shape().Size()); }
+
+ const Tensor& Values() const {
+ return values_;
+ }
+
+ const Tensor& Indices() const {
+ return indices_;
+ }
+
+ const TensorShape& Shape() const {
+ return shape_;
+ }
+
+ Tensor& MutableValues() {
+ return values_;
+ }
+
+ Tensor& MutableIndices() {
+ return indices_;
+ }
+
+ //TensorShape& MutableShape() {
+ // return shape_;
+ //}
+
+ private:
+ Tensor values_;
+ Tensor indices_;
+ TensorShape shape_; // The shape of corresponding dense-tensor.
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/tensor.h b/include/onnxruntime/core/framework/tensor.h
new file mode 100644
index 0000000..5c67796
--- /dev/null
+++ b/include/onnxruntime/core/framework/tensor.h
@@ -0,0 +1,246 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <stddef.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "gsl/gsl"
+#include "core/common/common.h"
+#include "core/framework/allocator.h"
+#include "core/framework/tensor_shape.h"
+#include "onnxruntime_config.h"
+#include "core/framework/data_types.h"
+#include "core/framework/data_types_internal.h"
+
+namespace onnxruntime {
+// TODO: Do we need this class or is IAllocator::MakeUniquePtr sufficient/better
+class BufferDeleter {
+ public:
+ BufferDeleter() : alloc_(nullptr) {}
+ BufferDeleter(AllocatorPtr alloc)
+ : alloc_(alloc) {}
+
+ void operator()(void* p) const {
+ if (alloc_)
+ alloc_->Free(p);
+ }
+
+ private:
+ // TODO: we may need consider the lifetime of alloc carefully
+ // The alloc_ here is the allocator that used to allocate the buffer
+ // And need go with the unique_ptr together. If it is using our internal
+ // allocator, it is ok as our allocators are global managed. But if it
+ // is provide by user, user need to be very careful about it.
+ // A weak_ptr may be a choice to reduce the impact, but that require to
+ // change our current allocator mgr to use shared_ptr. Will revisit it
+ // later.
+ AllocatorPtr alloc_;
+};
+
+using BufferUniquePtr = std::unique_ptr<void, BufferDeleter>;
+using BufferNakedPtr = void*;
+//TODO:ensure dtype_!=nullptr
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#ifdef HAS_NULL_DEREFERENCE
+#pragma GCC diagnostic ignored "-Wnull-dereference"
+#endif
+#endif
+/*
+ We want to keep tensor as simple as possible, it is just a placeholder
+ for a piece of memory, with additional shape information.
+ Memory is owned and managed by Executor / Workspace, so Tensor just uses
+ it, and won't do any allocation / release.
+*/
+class Tensor final {
+ public:
+ Tensor() = default; // to allow creating vector<Tensor> to support seq(tensor)
+
+ /**
+ * Create tensor with given type, shape, pre-allocated memory and allocator info.
+ * This function won't check if the preallocated buffer(p_data) has enough room for the shape.
+ * \param data A preallocated buffer. Can be NULL if the shape is empty.
+ * Tensor does not own the data and will not delete it
+ * \param alloc Where the buffer('data') was allocated from
+ * \param offset Offset in bytes to start of Tensor within p_data.
+ */
+ Tensor(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc,
+ ptrdiff_t offset = 0);
+
+ /**
+ * Deprecated. The orginal design is this Tensor class won't do any allocation / release.
+ * However, this function will allocate the buffer for the shape, and do placement new if p_type is string tensor.
+ */
+ Tensor(MLDataType p_type, const TensorShape& shape, std::shared_ptr<IAllocator> allocator);
+
+ ~Tensor();
+
+ //Move is allowed
+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(Tensor);
+
+ Tensor(Tensor&& other) noexcept;
+
+ Tensor& operator=(Tensor&& other) noexcept;
+
+ /**
+ Returns the data type.
+ */
+ MLDataType DataType() const { return dtype_; }
+
+ /**
+ Returns the data type enum constant
+ @remarks Use utils::ToTensorProtoElementType<T> for comparison.
+ */
+ int32_t GetElementType() const {
+ return dtype_->GetDataType();
+ }
+
+ // Check if contains string data. This is a separate
+ // interface bc it is frequently used.
+ bool IsDataTypeString() const {
+ return utils::IsPrimitiveDataType<std::string>(dtype_);
+ }
+
+ // Checks if the Tensor contains data type T
+ template <class T>
+ bool IsDataType() const {
+ return utils::IsPrimitiveDataType<T>(dtype_);
+ }
+
+ /**
+ Returns the shape of the tensor.
+ */
+ const TensorShape& Shape() const noexcept { return shape_; }
+
+ /**
+ Returns the location of the tensor's memory
+ */
+ const OrtMemoryInfo& Location() const { return alloc_info_; }
+
+ /**
+ May return nullptr if tensor size is zero
+ */
+ template <typename T>
+ T* MutableData() {
+ // Type check
+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",
+ "T ", "!=", dtype_);
+ return reinterpret_cast<T*>(static_cast<char*>(p_data_) + byte_offset_);
+ }
+
+ /**
+ May return nullptr if tensor size is zero
+ */
+ template <typename T>
+ gsl::span<T> MutableDataAsSpan() {
+ // Type check
+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",
+ "T ", "!=", dtype_);
+ T* data = reinterpret_cast<T*>(static_cast<char*>(p_data_) + byte_offset_);
+ return gsl::make_span(data, static_cast<size_t>(shape_.Size()));
+ }
+
+ template <typename T>
+ const T* Data() const {
+ // Type check
+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",
+ "T ", "!=", dtype_);
+ return reinterpret_cast<const T*>(static_cast<char*>(p_data_) + byte_offset_);
+ }
+
+ template <typename T>
+ gsl::span<const T> DataAsSpan() const {
+ // Type check
+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",
+ "T ", "!=", dtype_);
+ const T* data = reinterpret_cast<const T*>(static_cast<char*>(p_data_) + byte_offset_);
+ return gsl::make_span(data, static_cast<typename gsl::span<T>::index_type>(shape_.Size()));
+ }
+
+ void* MutableDataRaw(MLDataType type) {
+ ORT_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_);
+ return static_cast<char*>(p_data_) + byte_offset_;
+ }
+
+ const void* DataRaw(MLDataType type) const {
+ ORT_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_);
+ return static_cast<char*>(p_data_) + byte_offset_;
+ }
+
+ void* MutableDataRaw() noexcept {
+ return static_cast<char*>(p_data_) + byte_offset_;
+ }
+
+ const void* DataRaw() const noexcept {
+ return static_cast<char*>(p_data_) + byte_offset_;
+ }
+
+ bool OwnsBuffer() const noexcept {
+ return buffer_deleter_ != nullptr;
+ }
+
+ /**
+ * Resizes the tensor without touching underlying storage.
+ * This requires the total size of the tensor to remains constant.
+ * @warning this function is NOT thread-safe.
+ */
+ inline void Reshape(const TensorShape& new_shape) {
+ ORT_ENFORCE(shape_.Size() == new_shape.Size(),
+ "Tensor size (" + std::to_string(shape_.Size()) +
+ ") != new size (" + std::to_string(new_shape.Size()) + ")");
+ shape_ = new_shape;
+ }
+
+ /**
+ * Get the byte offset with respect to the p_data
+ * @warning this is a temporary solution for reusing the buffer bigger than needed.
+ * @warning use with caution - make sure you do boundary check before calling this method (see view.cc)
+ */
+ inline ptrdiff_t ByteOffset() const {
+ return byte_offset_;
+ }
+
+ /**
+ * Set the byte offset with respect to the p_data
+ * @warning this is a temporary solution for reusing the buffer bigger than needed.
+ */
+ inline void SetByteOffset(ptrdiff_t byte_offset) {
+ byte_offset_ = byte_offset;
+ }
+
+ /**
+ The number of bytes of data.
+ */
+ size_t SizeInBytes() const;
+
+ // More API methods.
+ private:
+ void Init(MLDataType p_type,
+ const TensorShape& shape,
+ void* p_raw_data,
+ AllocatorPtr deleter,
+ ptrdiff_t offset = 0);
+
+ void ReleaseBuffer();
+
+ void* p_data_;
+ /**
+ if buffer_deleter_ is null, it means tensor does not own the buffer.
+ otherwise tensor will use the deleter to release the buffer when
+ tensor is released.
+ */
+ AllocatorPtr buffer_deleter_;
+
+ TensorShape shape_;
+ const PrimitiveDataTypeBase* dtype_;
+ OrtMemoryInfo alloc_info_;
+ ptrdiff_t byte_offset_;
+};
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.h
new file mode 100644
index 0000000..9a2609b
--- /dev/null
+++ b/include/onnxruntime/core/framework/tensor_shape.h
@@ -0,0 +1,143 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <iosfwd>
+#include <vector>
+#include <algorithm>
+#include <string>
+#include <cstring>
+#include "onnxruntime_config.h"
+
+namespace onnxruntime {
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#ifdef HAS_NULL_DEREFERENCE
+#pragma GCC diagnostic ignored "-Wnull-dereference"
+#endif
+#endif
+class TensorShape : private std::vector<int64_t> {
+ // TODO - Use a custom STL allocator to avoid heap allocations in the common case.
+ // We use negative numbers for unknown symbolic dimension. Each negative
+ // number represents a unique symbolic dimension.
+ // Private inheritance is used to prevent ambiguity of element versus dimension size
+ public:
+ TensorShape() = default;
+
+ TensorShape(const TensorShape& /*other*/) = default;
+ TensorShape& operator=(const TensorShape& /*other*/) = default;
+
+ TensorShape(TensorShape&& /*other*/) = default;
+ TensorShape& operator=(TensorShape&& /*other*/) = default;
+
+ TensorShape(const std::vector<int64_t>& dims) : std::vector<int64_t>(dims) {}
+
+ TensorShape(std::vector<int64_t>&& dims) : std::vector<int64_t>(std::move(dims)) {}
+
+ TensorShape(const std::initializer_list<int64_t>& dims) : std::vector<int64_t>(dims) {}
+
+ TensorShape(const int64_t* dimension_sizes, size_t dimension_count);
+
+ TensorShape(const std::vector<int64_t>& dims, size_t start, size_t end);
+
+ /**
+ Return the dimension specified by <idx>.
+ */
+ const int64_t& operator[](size_t idx) const {
+ return std::vector<int64_t>::operator[](static_cast<int>(idx));
+ }
+
+ int64_t& operator[](size_t idx) {
+ return std::vector<int64_t>::operator[](static_cast<int>(idx));
+ }
+
+ bool operator==(const TensorShape& other) const noexcept {
+ auto thisVector = static_cast<const std::vector<int64_t>*>(this);
+ auto otherVector = static_cast<const std::vector<int64_t>*>(&other);
+ return *thisVector == *otherVector;
+ }
+
+ bool operator!=(const TensorShape& other) const noexcept {
+ return !(*this == other);
+ }
+
+ size_t NumDimensions() const noexcept {
+ return size();
+ }
+
+ /**
+ Copy dims into an array with given size
+ */
+ void CopyDims(int64_t* dims, size_t num_dims) const {
+ memcpy(dims, data(), sizeof(value_type) * std::min(num_dims, NumDimensions()));
+ }
+
+ /**
+ Return underlying vector representation.
+ */
+ const std::vector<int64_t>& GetDims() const { return *this; }
+
+ /**
+ * Return the total number of elements. Returns 1 for an empty (rank 0) TensorShape.
+ *
+ * May return -1
+ */
+ int64_t Size() const;
+
+ /**
+ Return the total number of elements up to the specified dimension.
+ If the dimension interval is empty (dimension == 0), return 1.
+ @param dimension Return size up to this dimension. Value must be between 0 and this->NumDimensions(), inclusive.
+ */
+ int64_t SizeToDimension(size_t dimension) const;
+
+ /**
+ Return the total number of elements from the specified dimension to the end of the tensor shape.
+ If the dimension interval is empty (dimension == this->NumDimensions()), return 1.
+ @param dimension Return size from this dimension to the end. Value must be between 0 and this->NumDimensions(),
+ inclusive.
+ */
+ int64_t SizeFromDimension(size_t dimension) const;
+
+ /**
+ Return a new TensorShape of the dimensions from dimstart to dimend.
+ */
+ TensorShape Slice(size_t dimstart, size_t dimend) const;
+
+ /**
+ Return a new TensorShape of the dimensions from dimstart to end.
+ */
+ TensorShape Slice(size_t dimstart) const;
+
+ /**
+ output dimensions nicely formatted
+ */
+ std::string ToString() const;
+
+ /**
+ Calculate size between start and end.
+ Assumes start and end are between 0 and this->NumDimensions(), inclusive, and that
+ start < end.
+ */
+ int64_t SizeHelper(size_t start, size_t end) const;
+
+ /**
+ empty shape or 1D shape (1) is regarded as scalar tensor
+ */
+ bool IsScalar() const {
+ size_t len = size();
+ return len == 0 || (len == 1 && operator[](0) == 1);
+ }
+
+ static const TensorShape& ReinterpretBaseType(const std::vector<int64_t>& dimensions) {
+ static_assert(sizeof(TensorShape) == sizeof(std::vector<int64_t>), "Size of TensorShape prevents safe casting from vector");
+ return *static_cast<const TensorShape*>(&dimensions);
+ }
+};
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+// operator<< to nicely output to a stream
+std::ostream& operator<<(std::ostream& out, const TensorShape& shape);
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/basic_types.h b/include/onnxruntime/core/graph/basic_types.h
new file mode 100644
index 0000000..7c1203f
--- /dev/null
+++ b/include/onnxruntime/core/graph/basic_types.h
@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include <unordered_map>
+#include <unordered_set>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <functional>
+
+namespace ONNX_NAMESPACE {
+class ValueInfoProto;
+class TensorProto;
+class SparseTensorProto;
+class TypeProto;
+class AttributeProto;
+// define types that would come from the ONNX library if we were building against it.
+#if defined(ORT_MINIMAL_BUILD)
+using OperatorSetVersion = int;
+#endif
+
+} // namespace ONNX_NAMESPACE
+
+namespace onnxruntime {
+using NodeIndex = size_t;
+using Version = int64_t;
+using NodeArgInfo = ONNX_NAMESPACE::ValueInfoProto;
+using InitializedTensorSet = std::unordered_map<std::string, const ONNX_NAMESPACE::TensorProto*>;
+using ArgNameToTypeMap = std::unordered_map<std::string, ONNX_NAMESPACE::TypeProto>;
+using ProviderType = const std::string&;
+// TODO - Evaluate switching the types below to support transparent comparators and enable
+// lookups based on gsl::cstring_span<> and std::string_view. This would reduces allocations
+// converting to std::string, but requires conversion to std::map<std::string, foo, std::less<>>
+// instead of std::unordered_map<std::string, foo, [std::less<foo>]>.
+
+using NodeAttributes = std::unordered_map<std::string, ONNX_NAMESPACE::AttributeProto>;
+class IOnnxRuntimeOpSchemaCollection;
+using IOnnxRuntimeOpSchemaCollectionPtr = std::shared_ptr<IOnnxRuntimeOpSchemaCollection>;
+} // namespace onnxruntime
+
+namespace onnxruntime {
+class OpKernel;
+class OpKernelInfo;
+
+using KernelCreateFn = std::function<OpKernel*(const OpKernelInfo& info)>;
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h
new file mode 100644
index 0000000..f723a39
--- /dev/null
+++ b/include/onnxruntime/core/graph/constants.h
@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+
+constexpr const char* kNoOp = "NoOp";
+constexpr const char* kConstant = "Constant";
+constexpr const char* kFunctionOp = "_kFunctionOp";
+constexpr const char* kConstantValue = "value";
+constexpr const char* kOnnxDomain = "";
+constexpr const char* kOnnxDomainAlias = "ai.onnx";
+constexpr const char* kMLDomain = "ai.onnx.ml";
+constexpr const char* kMSDomain = "com.microsoft";
+constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc";
+constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers";
+constexpr const char* kMSDmlDomain = "com.microsoft.dml";
+constexpr const char* kNGraphDomain = "com.intel.ai";
+constexpr const char* kMIGraphXDomain = "";
+constexpr const char* kVitisAIDomain = "com.xilinx";
+
+constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider";
+constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider";
+constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider";
+constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider";
+constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider";
+constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider";
+constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider";
+constexpr const char* kNnapiExecutionProvider = "NnapiExecutionProvider";
+constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider";
+constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider";
+constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider";
+constexpr const char* kAclExecutionProvider = "ACLExecutionProvider";
+constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider";
+constexpr const char* kRocmExecutionProvider = "ROCMExecutionProvider";
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/function.h b/include/onnxruntime/core/graph/function.h
new file mode 100644
index 0000000..cf9eefd
--- /dev/null
+++ b/include/onnxruntime/core/graph/function.h
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/common.h"
+#include "core/graph/indexed_sub_graph.h"
+
+namespace onnxruntime {
+class Graph;
+class Node;
+} // namespace onnxruntime
+
+namespace onnxruntime {
+
+/**
+@class Function
+Class representing a Function.
+*/
+class Function {
+ public:
+ virtual ~Function() = default;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Gets the OpSchema for the Function. */
+ virtual const ONNX_NAMESPACE::OpSchema& OpSchema() const = 0;
+#endif
+
+ /** Gets the Graph instance for the Function body subgraph. */
+ virtual const onnxruntime::Graph& Body() const = 0;
+};
+
+/**
+Create a new Function instance.
+@param graph The graph containing the Function.
+@param nodes_to_fuse the IndexedSubGraph to use for the Function.
+*/
+std::unique_ptr<Function> MakeFunction(const onnxruntime::Graph& graph,
+ const IndexedSubGraph& nodes_to_fuse,
+ const logging::Logger& logger);
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h
new file mode 100644
index 0000000..d1841ff
--- /dev/null
+++ b/include/onnxruntime/core/graph/graph.h
@@ -0,0 +1,1407 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <limits>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "core/common/common.h"
+#include "core/common/const_pointer_container.h"
+#include "core/common/path.h"
+#include "core/common/status.h"
+#include "core/common/logging/logging.h"
+#include "core/graph/basic_types.h"
+#include "core/graph/constants.h"
+#include "core/graph/graph_nodes.h"
+#include "core/graph/node_arg.h"
+#include "core/graph/onnx_protobuf.h"
+#include "core/graph/function.h"
+#include "gsl/gsl"
+
+namespace flatbuffers {
+class FlatBufferBuilder;
+template <typename T>
+struct Offset;
+} // namespace flatbuffers
+
+namespace onnxruntime {
+class Graph;
+struct IndexedSubGraph;
+class Model;
+class OpSignature;
+
+namespace experimental {
+namespace fbs {
+struct Graph;
+struct Node;
+struct NodeEdge;
+} // namespace fbs
+} // namespace experimental
+
+/**
+@class Node
+Class representing a node in the graph.
+*/
+class Node {
+ public:
+ /** Node types */
+ enum class Type {
+ Primitive = 0, ///< The node refers to a primitive operator.
+ Fused = 1, ///< The node refers to a function.
+ };
+
+ ~Node() = default;
+
+ /**
+ @class EdgeEnd
+ Class representing the end of an edge. It could be an input or output edge end of a node.
+ For the node's input edge end, it's the source end, as the destination end is the node itself.
+ For the node's output edge end, it's the destination end, as the source end is the node itself.
+ */
+ class EdgeEnd {
+ public:
+ /**
+ Construct an EdgeEnd
+ @param node The source node if this is an input edge to the current node,
+ or the destination node if this is an output edge from the current node.
+ @param src_arg_index The node arg index of source node of the edge.
+ @param dst_arg_index The node arg index of destination node of the edge.
+ */
+ EdgeEnd(const Node& node, int src_arg_index, int dst_arg_index) noexcept;
+
+ /** Construct a control edge.
+ @param node The node the edge joins to the current node.
+ */
+ explicit EdgeEnd(const Node& node) noexcept;
+
+ /** Gets the Node that this EdgeEnd refers to. */
+ const Node& GetNode() const noexcept { return *node_; }
+
+ /** Gets the source arg index.
+ @returns the source arg index of <*this> edge.*/
+ int GetSrcArgIndex() const { return src_arg_index_; }
+
+ /** Gets the destination arg index.
+ @returns the destination arg index of <*this> edge.*/
+ int GetDstArgIndex() const { return dst_arg_index_; }
+
+ private:
+ const Node* node_;
+ const int src_arg_index_;
+ const int dst_arg_index_;
+ };
+
+ /** Gets the Node's NodeIndex. */
+ NodeIndex Index() const noexcept { return index_; }
+
+ /** Gets the Node's name. */
+ const std::string& Name() const noexcept { return name_; }
+
+ /** Gets the Node's operator type. */
+ const std::string& OpType() const noexcept { return op_type_; }
+
+ /** Gets the domain of the OperatorSet that specifies the operator returned by #OpType. */
+ const std::string& Domain() const noexcept { return domain_; }
+
+ /** Gets the path of the owning model if any. */
+ const Path& ModelPath() const noexcept;
+
+ /** Gets the Node's execution priority.
+ @remarks Lower value means higher priority */
+ int Priority() const noexcept { return priority_; };
+
+ /** Sets the execution priority of a node.
+ @remarks Lower value means higher priority */
+ void SetPriority(int priority) noexcept;
+
+ /** Gets the node description. */
+ const std::string& Description() const noexcept { return description_; }
+
+ /** Gets the Node's Node::Type. */
+ Node::Type NodeType() const noexcept { return node_type_; }
+
+ /** Gets the opset version that the Node's operator was first defined in.
+ @returns Opset version. If -1 the Node's operator has not been set.
+ @remarks Prefer over Op()->SinceVersion() as Op() is disabled in a minimal build
+ */
+ int SinceVersion() const noexcept { return since_version_; }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Gets the Node's OpSchema.
+ @remarks The graph containing this node must be resolved, otherwise nullptr will be returned. */
+ const ONNX_NAMESPACE::OpSchema* Op() const noexcept { return op_; }
+
+ /**
+ Gets the function body if applicable otherwise nullptr
+ @param try_init_func_body If not already intialized, initialize the function body
+ (only applicable to operators which are defined as function in ONNX spec).
+ Function body can be initialized in 2 cases :
+ 1. For nodes of type "Fused"
+ 2. For nodes which are defined as functions in ONNX spec (example: DynamicQuantizeLinear)
+ For all other cases this will always return nullptr.
+ Nodes of type "Fused" are created during partitioning and the function body
+ initialization for such nodes also happens during node creation. Therefore,
+ initialization of function body will happen via this method only in case 2 mentioned above.
+ */
+ const Function* GetFunctionBody(bool try_init_func_body = true);
+
+ /** Gets the function body if applicable otherwise nullptr. */
+ const Function* GetFunctionBody() const noexcept { return func_body_; }
+
+#endif
+
+ /**
+ Helper to iterate through the container returned by #InputDefs() or #OutputDefs() and call the provided function.
+ @param node_args Collection of NodeArgs returned by #InputDefs() or #OutputDefs()
+ @param func Function to call for each valid NodeArg in the node_args. The function is called with the NodeArg
+ and the index number in the container.
+ @returns common::Status with success or error information.
+ @remarks Returns immediately on error.
+ */
+ static common::Status ForEachWithIndex(const ConstPointerContainer<std::vector<NodeArg*>>& node_args,
+ std::function<common::Status(const NodeArg& arg, size_t index)> func) {
+ for (size_t index = 0; index < node_args.size(); ++index) {
+ auto arg = node_args[index];
+ if (!arg->Exists())
+ continue;
+ ORT_RETURN_IF_ERROR(func(*arg, index));
+ }
+ return common::Status::OK();
+ }
+
+ /** Gets the count of arguments for each of the Node's explicit inputs. */
+ const std::vector<int>& InputArgCount() const noexcept { return definitions_.input_arg_count; }
+
+ /** Gets the Node's input definitions.
+ @remarks requires ConstPointerContainer wrapper to apply const to the NodeArg pointers so access is read-only. */
+ ConstPointerContainer<std::vector<NodeArg*>> InputDefs() const noexcept {
+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.input_defs);
+ }
+
+ /** Gets the implicit inputs to this Node.
+ If this Node contains a subgraph, these are the NodeArg's that are implicitly consumed by Nodes within that
+ subgraph. e.g. If and Loop operators.*/
+ ConstPointerContainer<std::vector<NodeArg*>> ImplicitInputDefs() const noexcept {
+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.implicit_input_defs);
+ }
+
+ /** Gets the Node's output definitions.
+ @remarks requires ConstPointerContainer wrapper to apply const to the NodeArg pointers so access is read-only. */
+ ConstPointerContainer<std::vector<NodeArg*>> OutputDefs() const noexcept {
+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.output_defs);
+ }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /**
+ Helper to iterate through the container returned by #MutableInputDefs() or #MutableOutputDefs() and call the provided function.
+ @param node_args Collection of NodeArgs returned by #MutableInputDefs() or #MutableOutputDefs()
+ @param func Function to call for each valid NodeArg in the node_args. The function is called with the NodeArg
+ and the index number in the container.
+ @returns common::Status with success or error information.
+ @remarks Returns immediately on error.
+ */
+ static common::Status ForEachMutableWithIndex(std::vector<NodeArg*>& node_args,
+ std::function<common::Status(NodeArg& arg, size_t index)> func) {
+ for (size_t index = 0; index < node_args.size(); ++index) {
+ auto arg = node_args[index];
+ if (!arg->Exists())
+ continue;
+ ORT_RETURN_IF_ERROR(func(*arg, index));
+ }
+ return common::Status::OK();
+ }
+ /** Gets a modifiable count of arguments for each of the Node's explicit inputs.
+ @todo This should be removed in favor of a method that updates the input args and the count.
+ Currently these operations are separate which is not a good setup. */
+ std::vector<int>& MutableInputArgsCount() { return definitions_.input_arg_count; }
+
+ /** Gets a modifiable collection of the Node's input definitions. */
+ std::vector<NodeArg*>& MutableInputDefs() noexcept {
+ return definitions_.input_defs;
+ }
+
+ /** Gets a modifiable collection of the Node's implicit input definitions. */
+ std::vector<NodeArg*>& MutableImplicitInputDefs() noexcept {
+ return definitions_.implicit_input_defs;
+ }
+
+ /** Gets a modifiable collection of the Node's output definitions. */
+ std::vector<NodeArg*>& MutableOutputDefs() noexcept {
+ return definitions_.output_defs;
+ }
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Struct to provide sorting between EdgeEnd instances based on NodeIndex first, and NodeArg::Name second. */
+ struct EdgeEndCompare {
+ bool operator()(const EdgeEnd& lhs, const EdgeEnd& rhs) const {
+ if (lhs.GetNode().Index() == rhs.GetNode().Index()) {
+ if (lhs.GetSrcArgIndex() == rhs.GetSrcArgIndex()) {
+ return lhs.GetDstArgIndex() < rhs.GetDstArgIndex();
+ }
+ return lhs.GetSrcArgIndex() < rhs.GetSrcArgIndex();
+ }
+ return lhs.GetNode().Index() < rhs.GetNode().Index();
+ }
+ };
+
+ using EdgeSet = std::set<EdgeEnd, EdgeEndCompare>;
+ using EdgeConstIterator = EdgeSet::const_iterator;
+
+ /**
+ @class NodeConstIterator
+ Class to provide const access to Node instances iterated via an EdgeConstIterator. */
+ class NodeConstIterator {
+ public:
+ NodeConstIterator(EdgeConstIterator p_iter);
+
+ bool operator==(const NodeConstIterator& p_other) const;
+
+ bool operator!=(const NodeConstIterator& p_other) const;
+
+ void operator++();
+ void operator--();
+
+ const Node& operator*() const;
+ const Node* operator->() const;
+
+ private:
+ EdgeConstIterator m_iter;
+ };
+
+ // Functions defined to traverse a Graph as below.
+
+ /** Gets an iterator to the beginning of the input nodes to this Node. */
+ NodeConstIterator InputNodesBegin() const noexcept { return NodeConstIterator(relationships_.input_edges.cbegin()); };
+ /** Gets an iterator to the end of the input nodes to this Node. */
+ NodeConstIterator InputNodesEnd() const noexcept { return NodeConstIterator(relationships_.input_edges.cend()); }
+
+ /** Gets an iterator to the beginning of the output nodes from this Node. */
+ NodeConstIterator OutputNodesBegin() const noexcept {
+ return NodeConstIterator(relationships_.output_edges.cbegin());
+ }
+
+ /** Gets an iterator to the end of the output nodes from this Node. */
+ NodeConstIterator OutputNodesEnd() const noexcept { return NodeConstIterator(relationships_.output_edges.cend()); }
+
+ /** Gets an iterator to the beginning of the input edges to this Node.
+ @remarks There are no nullptr entries in this collection. */
+ EdgeConstIterator InputEdgesBegin() const noexcept { return relationships_.input_edges.cbegin(); }
+
+ /** Gets an iterator to the end of the input edges to this Node. */
+ EdgeConstIterator InputEdgesEnd() const noexcept { return relationships_.input_edges.cend(); }
+
+ /** Gets an iterator to the beginning of the output edges from this Node.
+ @remarks There are no nullptr entries in this collection. */
+ EdgeConstIterator OutputEdgesBegin() const noexcept { return relationships_.output_edges.cbegin(); }
+
+ /** Gets an iterator to the end of the output edges from this Node. */
+ EdgeConstIterator OutputEdgesEnd() const noexcept { return relationships_.output_edges.cend(); }
+
+ /** Gets the Node's control inputs. */
+ const std::set<std::string>& ControlInputs() const noexcept { return relationships_.control_inputs; }
+
+ /** Gets the number of input edges to this Node */
+ size_t GetInputEdgesCount() const noexcept { return relationships_.input_edges.size(); }
+
+ /** Gets the number of output edges from this Node */
+ size_t GetOutputEdgesCount() const noexcept { return relationships_.output_edges.size(); }
+
+ /** Add an attribute to this Node with specified attribute name and value. */
+ void AddAttribute(const std::string& attr_name, const ONNX_NAMESPACE::AttributeProto& value);
+
+#define ADD_ATTR_INTERFACES(TypeName) \
+ void AddAttribute(const std::string& attr_name, const TypeName& value); \
+ void AddAttribute(const std::string& attr_name, \
+ const std::vector<TypeName>& values);
+
+ ADD_ATTR_INTERFACES(int64_t)
+ ADD_ATTR_INTERFACES(float)
+ ADD_ATTR_INTERFACES(std::string)
+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TensorProto)
+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::GraphProto)
+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::SparseTensorProto)
+
+ /** Gets the Node's attributes. */
+ const NodeAttributes& GetAttributes() const noexcept { return attributes_; }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Remove the specified attribute from this Node */
+ bool ClearAttribute(const std::string& attr_name);
+
+ /** Gets the Node's mutable attributes. */
+ NodeAttributes& GetMutableAttributes() noexcept { return attributes_; }
+
+ /** Gets the Graph instance that is instantiated from a GraphProto attribute during Graph::Resolve.
+ @param attr_name Attribute name for the GraphProto attribute.
+ @returns nullptr if the Graph instance has not been instantiated or attribute does not contain a GraphProto.
+ */
+ const Graph* GetGraphAttribute(const std::string& attr_name) const;
+
+ /** Gets the mutable Graph instance that is instantiated from a GraphProto attribute during Graph::Resolve.
+ @param attr_name Attribute name for the GraphProto attribute.
+ @returns nullptr if the Graph instance has not been instantiated or attribute does not contain a GraphProto.
+ */
+ Graph* GetMutableGraphAttribute(const std::string& attr_name);
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Checks if the Node contains at least one subgraph (this is the case for control flow operators, such as If, Scan, Loop).
+ @returns true if the Node contains a subgraph.
+ */
+ bool ContainsSubgraph() const {
+ return !attr_to_subgraph_map_.empty();
+ }
+
+ /** Get the const subgraphs from a node.
+ @remarks Creates a new vector so calling ContainsSubgraphs first is preferred. */
+ std::vector<gsl::not_null<const Graph*>> GetSubgraphs() const;
+
+ /** Gets a map of attribute name to the mutable Graph instances for all subgraphs of the Node.
+ @returns Map of the attribute name that defines the subgraph to the subgraph's Graph instance.
+ nullptr if the Node has no subgraphs.
+ */
+ const std::unordered_map<std::string, gsl::not_null<Graph*>>& GetAttributeNameToMutableSubgraphMap() {
+ return attr_to_subgraph_map_;
+ }
+
+ /** Gets the execution ProviderType that this node will be executed by. */
+ ProviderType GetExecutionProviderType() const noexcept { return execution_provider_type_; }
+
+ /** Sets the execution ProviderType that this Node will be executed by. */
+ void SetExecutionProviderType(ProviderType execution_provider_type) {
+ execution_provider_type_ = execution_provider_type;
+ }
+
+ /** Call the provided function for all explicit inputs, implicit inputs, and outputs of this Node.
+ If the NodeArg is an explicit or implicit input, is_input will be true when func is called.
+ @param include_missing_optional_defs Include NodeArgs that are optional and were not provided
+ i.e. NodeArg::Exists() == false.
+ */
+ void ForEachDef(std::function<void(const onnxruntime::NodeArg&, bool is_input)> func,
+ bool include_missing_optional_defs = false) const;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Replaces any matching definitions in the Node's explicit inputs or explicit outputs.
+ @param replacements Map of current NodeArg to replacement NodeArg.
+ */
+ void ReplaceDefs(const std::map<const onnxruntime::NodeArg*, onnxruntime::NodeArg*>& replacements);
+
+ /** Gets the NodeProto representation of this Node.
+ @param update_subgraphs Update the GraphProto values for any subgraphs in the returned NodeProto.
+ If graph optimization has been run this is most likely required
+ to ensure the complete Graph is valid.
+ */
+ void ToProto(ONNX_NAMESPACE::NodeProto& proto, bool update_subgraphs = false) const;
+
+ Status SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
+ flatbuffers::Offset<onnxruntime::experimental::fbs::Node>& fbs_node) const;
+
+ flatbuffers::Offset<onnxruntime::experimental::fbs::NodeEdge>
+ SaveEdgesToOrtFormat(flatbuffers::FlatBufferBuilder& builder) const;
+
+#endif
+
+#if defined(ENABLE_ORT_FORMAT_LOAD)
+ static Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Node& fbs_node, Graph& graph,
+ const logging::Logger& logger, std::unique_ptr<Node>& node);
+
+ Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Node& fbs_node, const logging::Logger& logger);
+ Status LoadEdgesFromOrtFormat(const onnxruntime::experimental::fbs::NodeEdge& fbs_node_edgs, const Graph& graph);
+#endif
+
+ /**
+ @class Definitions
+ The input and output definitions for this Node.
+ */
+ class Definitions {
+ public:
+ Definitions() = default;
+
+ /** The Node's explicit input definitions. */
+ std::vector<NodeArg*> input_defs;
+
+ /**
+ The number of inputs for each argument of the operator or function which this node refers.
+ @remarks For example, #input_defs has 10 elements (inputs), and #input_arg_count is {4, 6}.
+ This means that 4 elements (inputs) of input_defs map to the first argument of the operator or function, and
+ the other 6 map to the second argument.
+ */
+ std::vector<int> input_arg_count;
+
+ /** The Node's output definitions. */
+ std::vector<NodeArg*> output_defs;
+
+ /** The Node's implicit input definitions if the Node contains one or more subgraphs
+ (i.e. GraphProto attributes) and the subgraph/s implicitly consume these values.
+ @remarks For example, a subgraph in an 'If' node gets all its input values via this mechanism rather than
+ there being explicit inputs to the 'If' node that are passed to the subgraph.
+ They are pseudo-inputs to this Node as it has an implicit dependency on them. */
+ std::vector<NodeArg*> implicit_input_defs;
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Definitions);
+ };
+
+ /**
+ @class Relationships
+ Defines the relationships between this Node and other Nodes in the Graph.
+ */
+ class Relationships {
+ public:
+ Relationships() = default;
+
+ void Clear() noexcept {
+ input_edges.clear();
+ output_edges.clear();
+ control_inputs.clear();
+ }
+
+ /** The edges for Nodes that provide inputs to this Node. */
+ EdgeSet input_edges;
+
+ /** The edges for Nodes that receive outputs from this Node. */
+ EdgeSet output_edges;
+
+ /** The Node names of the control inputs to this Node. */
+ std::set<std::string> control_inputs;
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Relationships);
+ };
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Node);
+
+ // NOTE: This friendship relationship should ONLY be used for calling methods of the Node class and not accessing
+ // the data members directly, so that the Node can maintain its internal invariants.
+ friend class Graph;
+
+ Node(NodeIndex index, Graph& graph) : index_(index), graph_(&graph) {}
+
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
+ void Init(const std::string& name,
+ const std::string& op_type,
+ const std::string& description,
+ const std::vector<NodeArg*>& input_args,
+ const std::vector<NodeArg*>& output_args,
+ const NodeAttributes* attributes,
+ const std::string& domain);
+
+ // internal only method to allow selected classes to directly alter the input/output definitions and arg counts
+ Definitions& MutableDefinitions() noexcept;
+
+ // internal only method to allow selected classes to directly alter the links between nodes.
+ Relationships& MutableRelationships() noexcept;
+
+ void SetNodeType(Node::Type node_type) noexcept { node_type_ = node_type; }
+#endif
+
+ // create a Graph instance for an attribute that contains a GraphProto
+ void CreateSubgraph(const std::string& attr_name);
+
+ const std::vector<std::unique_ptr<Graph>>& MutableSubgraphs() noexcept { return subgraphs_; }
+
+ // validate and update the input arg count
+ common::Status UpdateInputArgCount();
+
+ void SetFunctionBody(const Function& func);
+
+ const Definitions& GetDefinitions() const noexcept { return definitions_; }
+ const Relationships& GetRelationships() const noexcept { return relationships_; }
+
+ // Node index. Default to impossible value rather than 0.
+ NodeIndex index_ = std::numeric_limits<NodeIndex>::max();
+
+ // Node name.
+ std::string name_;
+
+ // Node operator type.
+ std::string op_type_;
+
+ // OperatorSet domain of op_type_.
+ std::string domain_;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ // OperatorSchema that <*this> node refers to.
+ const ONNX_NAMESPACE::OpSchema* op_ = nullptr;
+#endif
+
+ // Execution priority, lower value for higher priority
+ int priority_ = 0;
+
+ // set from op_->SinceVersion() or via deserialization when OpSchema is not available
+ int since_version_ = -1;
+
+ Node::Type node_type_ = Node::Type::Primitive;
+
+ // The function body is owned by graph_
+ const Function* func_body_ = nullptr;
+
+ // Node doc string.
+ std::string description_;
+
+ // input/output defs and arg count
+ Definitions definitions_;
+
+ // Relationships between this node and others in the graph
+ Relationships relationships_;
+
+ // Device.
+ std::string execution_provider_type_;
+
+ // Map from attribute name to attribute.
+ // This allows attribute adding and removing.
+ NodeAttributes attributes_;
+
+ // Graph that contains this Node
+ Graph* graph_;
+
+ // Map of attribute name to the Graph instance created from the GraphProto attribute
+ std::unordered_map<std::string, gsl::not_null<Graph*>> attr_to_subgraph_map_;
+
+ // Graph instances for subgraphs that are owned by this Node
+ std::vector<std::unique_ptr<Graph>> subgraphs_;
+};
+
+/**
+@class Graph
+The Graph representation containing the graph inputs and outputs, the Node instances,
+and the edges connecting the nodes.
+*/
+class Graph {
+ public:
+ /** Gets the Graph name. */
+ const std::string& Name() const noexcept;
+
+ /** Gets the Graph description. */
+ const std::string& Description() const noexcept;
+
+ /** Gets the path of the owning model, if any. */
+ const Path& ModelPath() const;
+
+ /** Returns true if this is a subgraph or false if it is a high-level graph. */
+ bool IsSubgraph() const { return parent_graph_ != nullptr; }
+
+ /** Returns the parent graph if this is a subgraph */
+ const Graph* ParentGraph() const { return parent_graph_; }
+
+ /** Returns the mutable parent graph if this is a subgraph */
+ Graph* MutableParentGraph() { return parent_graph_; }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Sets the Graph name. */
+ void SetName(const std::string& name);
+
+ /** Gets the Graph description. */
+ void SetDescription(const std::string& description);
+
+ /** Add an initializer tensor to the Graph. */
+ void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor_proto);
+
+ /** Replaces the initializer tensor with the same name as the given initializer tensor.
+ The replacement initializer tensor must have the same type and shape as the existing initializer tensor.
+
+ Note: This currently has linear time complexity. There is room for improvement but it would likely require changes to
+ how initializer tensors are stored and tracked.
+ */
+ common::Status ReplaceInitializedTensor(const ONNX_NAMESPACE::TensorProto& new_initializer);
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Remove the initializer tensor with the provided name from the Graph. */
+ void RemoveInitializedTensor(const std::string& tensor_name);
+
+ /** Check if a given name is an initializer tensor's name in this graph. */
+ bool IsInitializedTensor(const std::string& name) const;
+
+ /** Gets an initializer tensor with the provided name.
+ @param[out] value Set to the TensorProto* if the initializer is found, or nullptr if not.
+ @returns True if found.
+ */
+ bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const;
+
+ /** Gets all the initializer tensors in this Graph. */
+ const InitializedTensorSet& GetAllInitializedTensors() const noexcept { return name_to_initial_tensor_; }
+
+ /** Removes all initializer tensors from this Graph and releases the memory they were using. */
+ void CleanAllInitializedTensors() noexcept;
+
+ /** Returns true if an initializer value can be overridden by a graph input with the same name. */
+ bool CanOverrideInitializer() const noexcept { return ir_version_ >= 4; }
+
+ /** returns the initializer's TensorProto if 'name' is an initializer, is constant and
+ cannot be overridden at runtime. If the initializer is not found or is not constant, a nullptr is returned.
+ @param check_outer_scope If true and the graph is a subgraph,
+ check ancestor graph/s for 'name' if not found in 'graph'.
+ @remarks check_outer_scope of true is not supported in a minimal build
+ */
+ const ONNX_NAMESPACE::TensorProto* GetConstantInitializer(const std::string& name, bool check_outer_scope) const;
+
+ /** Gets the Graph inputs excluding initializers.
+ These are the required inputs to the Graph as the initializers can be optionally overridden via graph inputs.
+ @remarks Contains no nullptr values. */
+ const std::vector<const NodeArg*>& GetInputs() const noexcept { return graph_inputs_excluding_initializers_; }
+
+ /** Gets the Graph inputs including initializers.
+ This is the full set of inputs, in the same order as defined in the GraphProto.
+ @remarks Contains no nullptr values. */
+ const std::vector<const NodeArg*>& GetInputsIncludingInitializers() const noexcept {
+ return graph_inputs_including_initializers_;
+ }
+
+ /** Return true if "node_arg" is a input or an initializer. Otherwise, returns false. */
+ bool IsInputsIncludingInitializers(const NodeArg* node_arg) const noexcept {
+ return std::find(graph_inputs_including_initializers_.begin(), graph_inputs_including_initializers_.end(), node_arg) != graph_inputs_including_initializers_.end();
+ }
+
+ /** Gets the Graph inputs that are initializers
+ These are overridable initializers. This is a difference between
+ graph_inputs_including_initializers_ and graph_inputs_excluding_initializers_
+ @remarks Contains no nullptr values. */
+ const std::vector<const NodeArg*>& GetOverridableInitializers() const {
+ return graph_overridable_initializers_;
+ }
+
+ /** Gets the Graph outputs.
+ @remarks Contains no nullptr values.*/
+ const std::vector<const NodeArg*>& GetOutputs() const noexcept { return graph_outputs_; }
+
+ bool IsOutput(const NodeArg* node_arg) const noexcept {
+ return std::find(graph_outputs_.begin(), graph_outputs_.end(), node_arg) != graph_outputs_.end();
+ }
+
+ /** Returns a vector with the indexes of the outputs of the given Node that are also Graph outputs. */
+ std::vector<int> GetNodeOutputsInGraphOutputs(const Node& node) const {
+ int output_idx = 0;
+ std::vector<int> indexes;
+ for (auto output_def : node.OutputDefs()) {
+ if (std::find(GetOutputs().cbegin(), GetOutputs().cend(), output_def) != GetOutputs().cend()) {
+ indexes.push_back(output_idx);
+ }
+
+ ++output_idx;
+ }
+
+ return indexes;
+ }
+
+ /** Gets the NodeArgs that represent value_info instances in the Graph.
+ These are the values that are neither Graph inputs nor outputs.
+ @remarks Contains no nullptr values. */
+ const std::vector<const NodeArg*>& GetValueInfo() const noexcept { return value_info_; }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ void AddValueInfo(const NodeArg* new_value_info);
+#endif
+
+ /** Gets the Node with the specified node index.
+ @returns Node instance if found. nullptr if node_index is invalid or node has been freed.
+ */
+ const Node* GetNode(NodeIndex node_index) const { return NodeAtIndexImpl(node_index); }
+
+ /** Gets the mutable Node with the specified node index.
+ @returns Mutable Node instance if found. nullptr if node_index is invalid or node has been freed.
+ */
+ Node* GetNode(NodeIndex node_index) { return NodeAtIndexImpl(node_index); }
+
+ /** Get a GraphNodes instance that provides mutable access to all valid Nodes in the Graph. */
+ GraphNodes& Nodes() noexcept { return iterable_nodes_; }
+
+ /** Get a GraphNodes instance that provides const access to all valid Nodes in the Graph. */
+ const GraphNodes& Nodes() const noexcept { return iterable_nodes_; }
+
+ /** Get a ConstGraphNodes instance that provides access to a filtered set of valid Nodes in the Graph.
+ @remarks We can't use GraphNodes as that would provide mutable access to the nodes by default, and we can't prevent
+ that by returning a const instance of GraphNodes as we're creating a new instance here due to the filter
+ being something we don't control (i.e. we have to return a new instance so it can't be const).
+ */
+ ConstGraphNodes FilteredNodes(GraphNodes::NodeFilterFunc&& filter_func) const noexcept {
+ return ConstGraphNodes(nodes_, std::move(filter_func));
+ }
+
+ /** Gets the maximum NodeIndex value used in the Graph. */
+ int MaxNodeIndex() const noexcept { return static_cast<int>(nodes_.size()); } //assume the casting won't overflow
+
+ /** Gets the number of valid Nodes in the Graph.
+ @remarks This may be smaller than MaxNodeIndex(), as Nodes may be removed during optimization.
+ */
+ int NumberOfNodes() const noexcept { return num_of_nodes_; }
+
+ /** Gets the mutable NodeArg with the provided name.
+ @returns Pointer to NodeArg if found, nullptr if not. */
+ NodeArg* GetNodeArg(const std::string& name) {
+ auto iter = node_args_.find(name);
+ if (iter != node_args_.end()) {
+ return iter->second.get();
+ }
+ return nullptr;
+ }
+
+ /** Gets the const NodeArg with the provided name.
+ @returns Pointer to const NodeArg if found, nullptr if not. */
+ const NodeArg* GetNodeArg(const std::string& name) const {
+ return const_cast<Graph*>(this)->GetNodeArg(name);
+ }
+
+ // search this and up through any parent_graph_ instance for a NodeArg
+ NodeArg* GetNodeArgIncludingParentGraphs(const std::string& node_arg_name);
+
+ /** Gets a mutable NodeArg by name. Creates a new NodeArg that is owned by this Graph if not found.
+ @param name The NodeArg name.
+ @param[in] p_arg_type Optional TypeProto to use if the NodeArg needs to be created.
+ @returns NodeArg reference.
+ */
+ NodeArg& GetOrCreateNodeArg(const std::string& name, const ONNX_NAMESPACE::TypeProto* p_arg_type) {
+ auto iter = node_args_.find(name);
+ if (iter != node_args_.end()) {
+ return *(iter->second);
+ }
+ auto result = node_args_.insert(std::make_pair(name, onnxruntime::make_unique<NodeArg>(name, p_arg_type)));
+ return *(result.first->second);
+ }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Generate a unique name in this Graph for a NodeArg */
+ std::string GenerateNodeArgName(const std::string& base_name);
+
+ /** Generate a unique name in this Graph for a Node */
+ std::string GenerateNodeName(const std::string& base_name);
+
+ /** Copy a Node and add it to this Graph.
+ @param other Node to copy
+ @returns Reference to the Node that was created and added to this Graph.
+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.
+ */
+ Node& AddNode(const Node& other);
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
+ /** Add a Node to this Graph.
+ @param name The Node name. Must be unique in this Graph.
+ @param op_type The operator type. e.g. ONNX operator name.
+ @param description Arbitrary description of the Node.
+ @param input_args The explicit inputs to this Node.
+ @param output_args The outputs from this Node.
+ @param attributes Optional NodeAttributes to add.
+ @param domain The domain for the op_type.
+ @returns Reference to the new Node.
+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.
+ */
+ Node& AddNode(const std::string& name,
+ const std::string& op_type,
+ const std::string& description,
+ const std::vector<NodeArg*>& input_args,
+ const std::vector<NodeArg*>& output_args,
+ const NodeAttributes* attributes = nullptr,
+ const std::string& domain = "");
+
+ /** Remove a Node from this Graph and free it.
+ The output edges of this specified node MUST have been removed before removing the node.
+ The input edges of this specified node is removed while removing the node. The process of
+ removing a node from a graph should be,
+ 1. Remove out edges of this specified node.
+ 2. Remove this specified node.
+ 3. Add new input edges connected with all out nodes.
+ @returns true if the node_index was valid
+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.
+ */
+ bool RemoveNode(NodeIndex node_index);
+
+ /** Add an edge between two Nodes.
+ @param src_node_index NodeIndex of source Node that is providing output to the destination Node.
+ @param dst_node_index NodeIndex of destination Node that is receiving input from the source Node.
+ @param src_arg_index node arg index of source node.
+ @param dst_arg_index node arg index of destination node.
+ */
+ void AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_arg_index, int dst_arg_index);
+
+ /** Remove an edge between two Nodes.
+ @param src_node_index NodeIndex of source Node to remove an output edge from.
+ @param dst_node_index NodeIndex of destination Node to remove an input edge from.
+ @param src_arg_index node arg index of source node.
+ @param dst_arg_index node arg index of destination node.
+ */
+ void RemoveEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_arg_index, int dst_arg_index);
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /**
+ Add a control edge between two Nodes in this Graph.
+ The source Node does not produce output that is directly consumed by the destination Node, however the
+ destination Node must execute after the source node. The control edge allows this ordering to occur.
+ */
+ bool AddControlEdge(NodeIndex src_node_index, NodeIndex dst_node_index);
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Mark the Graph as needing Resolve() to be called.
+ This should be done after modifying any aspect of the Graph that changes the Nodes or relationships between them. */
+ Graph& SetGraphResolveNeeded() noexcept {
+ graph_resolve_needed_ = true;
+ return *this;
+ }
+
+ /** Gets flag indicating whether Graph::Resolve needs to be called before using the Graph. */
+ bool GraphResolveNeeded() const noexcept {
+ return graph_resolve_needed_;
+ }
+
+ /** Sets flag that Graph::graph_proto_ needs to be updated to reflect changes in the Graph. */
+ Graph& SetGraphProtoSyncNeeded() noexcept {
+ graph_proto_sync_needed_ = true;
+ return *this;
+ }
+
+ /** Gets flag indicating whether Graph::graph_proto_ needs to be synchronized with this Graph instance. */
+ bool GraphProtoSyncNeeded() const noexcept {
+ return graph_proto_sync_needed_;
+ }
+
+ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,
+ up to their source node/s.
+ @param from NodeIndex values for a set of Nodes to traverse from.
+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.
+ @param leave Visit function invoked on the node after its parents have all been visited.
+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.
+ */
+ void ReverseDFSFrom(const std::vector<NodeIndex>& from,
+ const std::function<void(const Node*)>& enter,
+ const std::function<void(const Node*)>& leave,
+ const std::function<bool(const Node*, const Node*)>& comp = {}) const;
+
+ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,
+ up to their source node/s.
+ @param from Set of Nodes to traverse from.
+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.
+ @param leave Visit function invoked on the node after its parents have all been visited.
+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.
+ */
+ void ReverseDFSFrom(const std::vector<const Node*>& from,
+ const std::function<void(const Node*)>& enter,
+ const std::function<void(const Node*)>& leave,
+ const std::function<bool(const Node*, const Node*)>& comp = {}) const;
+
+ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,
+ up to their source node/s.
+ @param from Set of Nodes to traverse from.
+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.
+ @param leave Visit function invoked on the node after its parents have all been visited.
+ @param stop Stop traversal from node n to input node p if stop(n, p) is true.
+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.
+ */
+ void ReverseDFSFrom(const std::vector<const Node*>& from,
+ const std::function<void(const Node*)>& enter,
+ const std::function<void(const Node*)>& leave,
+ const std::function<bool(const Node*, const Node*)>& comp,
+ const std::function<bool(const Node*, const Node*)>& stop) const;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Performs topological sort with Kahn's algorithm on the graph/s.
+ @param enter Visit function that will be invoked on a node when it is visited.
+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.
+ */
+ void KahnsTopologicalSort(const std::function<void(const Node*)>& enter,
+ const std::function<bool(const Node*, const Node*)>& comp) const;
+
+#endif
+
+ /** Gets the map of operator domains to their opset versions. */
+ const std::unordered_map<std::string, int>& DomainToVersionMap() const noexcept {
+ return domain_to_version_;
+ }
+
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
+ /**
+ Create a single Node that will be the result of the a fusion of multiple nodes in this Graph.
+ @param sub_graph A IndexSubGraph instance with details of the nodes to fuse.
+ @param fused_node_name The name for the new Node.
+ @returns Node with fused subgraph.
+ @remarks As a new Graph instance for the fused nodes is not created, a GraphViewer can be constructed with the
+ IndexedSubGraph information to provide a view of the subgraph. The original nodes are left in place
+ while this is in use.
+ Call FinalizeFuseSubGraph to remove them once the fused replacement node is fully created.
+ */
+ Node& BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);
+
+ void FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node);
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Gets the GraphProto representation of this Graph. */
+ const ONNX_NAMESPACE::GraphProto& ToGraphProto();
+ ONNX_NAMESPACE::GraphProto ToGraphProto() const;
+
+ /** Gets the ISchemaRegistry instances being used with this Graph. */
+ IOnnxRuntimeOpSchemaCollectionPtr GetSchemaRegistry() const;
+
+ /**
+ Create a single Function based Node that is the result of the a fusion of multiple nodes in this Graph.
+ A new Graph instance will be created for the fused nodes.
+ @param sub_graph A IndexSubGraph instance with details of the nodes to fuse. Ownership is transferred to the new Node
+ @param fused_node_name The name for the new Node.
+ @returns Function based Node with fused subgraph. The Node body will contain a Function instance.
+ */
+ Node& FuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);
+
+ /**
+ Directly insert the nodes in the function Node provided into this Graph.
+ @param node Node with Node::Type of Node::Type::Fused
+ @returns Status indicating success or providing an error message.
+ */
+ Status InlineFunction(Node& node);
+
+ /** Initialize function body for the given node */
+ void InitFunctionBodyForNode(Node& node);
+
+ /** Mark a NodeArg name as coming from the outer scope when programmatically constructing a Graph that will
+ be used as a GraphProto attribute in another Node..
+ e.g. when creating a Graph instance that will be used as a subgraph in a control flow operator, it is necessary to
+ define placeholder NodeArgs for outer scope values. This prevents these values from becoming explicit graph inputs
+ when the Graph is resolved.
+ */
+ void AddOuterScopeNodeArg(const std::string& name) {
+ ORT_IGNORE_RETURN_VALUE(outer_scope_node_arg_names_.insert(name));
+ }
+
+ /** Explicitly set graph inputs.
+ @param inputs NodeArgs that represent complete graph inputs which need to be explicitly ordered.
+ @remarks Note that the input order matters for subgraphs.
+ */
+ void SetInputs(const std::vector<const NodeArg*>& inputs);
+
+ /** Explicitly set graph outputs.
+ @param outputs NodeArgs that represent complete graph outputs which need to be explicitly ordered.
+ @remarks Note that the output order matters for subgraphs.
+ */
+ void SetOutputs(const std::vector<const NodeArg*>& outputs);
+
+ /** Sets the type of a NodeArg, replacing existing type/shape if any */
+ void SetNodeArgType(NodeArg& arg, const onnx::TypeProto& type_proto);
+
+ const Node* GetProducerNode(const std::string& node_arg_name) const {
+ return GetProducerNodeImpl(*this, node_arg_name);
+ }
+
+ Node* GetMutableProducerNode(const std::string& node_arg_name) {
+ return GetProducerNodeImpl(*this, node_arg_name);
+ }
+
+ void UpdateProducerNode(const std::string& node_arg_name, NodeIndex node_index) {
+ auto iter = node_arg_to_producer_node_.find(node_arg_name);
+
+ if (iter != node_arg_to_producer_node_.end()) {
+ iter->second = node_index;
+ } else {
+ node_arg_to_producer_node_[node_arg_name] = node_index;
+ }
+ }
+
+ std::vector<const Node*> GetConsumerNodes(const std::string& node_arg_name) const {
+ return GetConsumerNodesImpl(*this, node_arg_name);
+ }
+
+ std::vector<Node*> GetMutableConsumerNodes(const std::string& node_arg_name) {
+ return GetConsumerNodesImpl(*this, node_arg_name);
+ }
+
+ void UpdateConsumerNodes(const std::string& node_arg_name, const std::vector<Node*>& nodes) {
+ auto iter = node_arg_to_consumer_nodes_.find(node_arg_name);
+ if (iter != node_arg_to_consumer_nodes_.end()) {
+ node_arg_to_consumer_nodes_.erase(node_arg_name);
+ }
+ for (Node* node : nodes) {
+ node_arg_to_consumer_nodes_[node_arg_name].insert(node->Index());
+ }
+ }
+
+ /** During constant folding it may become possible to infer the shape for a node.
+ To avoid running a full Resolve allow an individual node to have the shape inferencing re-run.
+ */
+ Status UpdateShapeInference(Node& node);
+
+ // Options to control Graph::Resolve.
+ struct ResolveOptions {
+ // Whether to override existing types with inferred types.
+ bool override_types = false;
+ // Names of initializers to keep even if unused (optional).
+ const std::unordered_set<std::string>* initializer_names_to_preserve = nullptr;
+ // Whether to set that no proto sync is required after resolving.
+ // Useful for resolving right after loading from a GraphProto.
+ bool no_proto_sync_required = false;
+ };
+
+ /**
+ Resolve this Graph to ensure it is completely valid, fully initialized, and able to be executed.
+ 1. Run through all validation rules.
+ a. Node name and node output's names should be unique.
+ b. Attribute match between node and op definition.
+ c. Input/Output match between node and op definition.
+ d. Graph is acyclic and sort nodes in topological order.
+ 2. Check & Setup inner nodes' dependency.
+ 3. Cleanup function definition lists.
+ Note: the weights for training can't be cleaned during resolve.
+ @returns common::Status with success or error information.
+ */
+ common::Status Resolve(const ResolveOptions& options);
+
+ common::Status Resolve() {
+ ResolveOptions default_options;
+ return Resolve(default_options);
+ }
+
+ common::Status SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,
+ flatbuffers::Offset<onnxruntime::experimental::fbs::Graph>& fbs_graph) const;
+
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Returns the Node containing the GraphProto for this Graph instance if IsSubgraph is true */
+ const Node* ParentNode() const { return parent_node_; }
+
+ /** Returns true if the name is for a value that is coming from outer scope */
+ bool IsOuterScopeValue(const std::string& name) const {
+ if (!parent_node_) return false;
+ const auto& implicit_input_defs = parent_node_->ImplicitInputDefs();
+ return std::any_of(implicit_input_defs.cbegin(), implicit_input_defs.cend(),
+ [&name](const NodeArg* implicit_input) {
+ return implicit_input->Name() == name;
+ });
+ }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Construct a Graph instance for a subgraph that is created from a GraphProto attribute in a Node.
+ Inherits some properties from the parent graph.
+ @param parent_graph The Graph containing the Node that has the GraphProto attribute.
+ @param parent_node The Node that has the GraphProto attribute.
+ @param subgraph_proto The GraphProto from the Node attribute.
+ */
+ Graph(Graph& parent_graph, const Node& parent_node, ONNX_NAMESPACE::GraphProto& subgraph_proto);
+#endif
+
+ virtual ~Graph();
+
+#if defined(ENABLE_ORT_FORMAT_LOAD)
+ static common::Status LoadFromOrtFormat(
+ const onnxruntime::experimental::fbs::Graph& fbs_graph, const Model& owning_model,
+ const std::unordered_map<std::string, int>& domain_to_version,
+ const logging::Logger& logger, std::unique_ptr<Graph>& graph);
+
+ // deserialize a subgraph
+ static Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Graph& fbs_graph,
+ Graph& parent_graph, const Node& parent_node,
+ const logging::Logger& logger, std::unique_ptr<Graph>& graph);
+#endif
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Graph);
+
+ // This friendship relationship should only be used to call Graph::Graph and
+ // Graph::LoadGraph All other access should be via the public API.
+ friend class Model;
+
+ Graph() = delete;
+
+ // Create empty Graph instance to re-create from ORT format serialized data.
+ Graph(const Model& owning_model,
+ const std::unordered_map<std::string, int>& domain_to_version,
+ Graph* parent_graph, const Node* parent_node,
+ const logging::Logger& logger);
+
+#if defined(ENABLE_ORT_FORMAT_LOAD)
+ // Populate Graph instance from ORT format serialized data.
+ common::Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Graph& fbs_graph);
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD)
+ // Constructor: Given a <GraphProto> loaded from model file, construct
+ // a <Graph> object. Used by Model to create a Graph instance.
+ Graph(const Model& owning_model,
+ ONNX_NAMESPACE::GraphProto* graph_proto,
+ const std::unordered_map<std::string, int>& domain_to_version,
+ Version ir_version,
+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry,
+ const logging::Logger& logger);
+
+ // internal use by the Graph class only
+ Graph(const Model& owning_model,
+ ONNX_NAMESPACE::GraphProto* graph_proto,
+ const std::unordered_map<std::string, int>& domain_to_version,
+ Version ir_version,
+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry,
+ Graph* parent_graph,
+ const Node* parent_node,
+ const logging::Logger& logger);
+
+ void InitializeStateFromModelFileGraphProto();
+
+ // Add node with specified <node_proto>.
+ Node& AddNode(const ONNX_NAMESPACE::NodeProto& node_proto,
+ const ArgNameToTypeMap& name_to_type);
+
+#endif
+
+ Version IrVersion() const noexcept {
+ return ir_version_;
+ }
+
+ Graph& GraphResolveNeeded(bool needed) noexcept {
+ graph_resolve_needed_ = needed;
+ return *this;
+ }
+
+ Graph& GraphProtoSyncNeeded(bool needed) noexcept {
+ graph_proto_sync_needed_ = needed;
+ return *this;
+ }
+
+ // During the Resolve of a Graph it is necessary to recursively descend into subgraphs (created from GraphProto
+ // Node attributes in the Graph) if present.
+ // The ResolveContext holds the collection of values for the current Graph instance, be it the main graph
+ // or a subgraph, so that the various operations that are part of the Resolve can work iteratively or
+ // recursively as needed.
+ struct ResolveContext {
+ ResolveContext() = default;
+
+ std::unordered_map<std::string, std::pair<Node*, int>> output_args;
+ std::unordered_set<std::string> inputs_and_initializers;
+ std::unordered_set<std::string> outer_scope_node_args;
+ std::unordered_map<std::string, NodeIndex> node_name_to_index;
+ std::unordered_set<Node*> nodes_with_subgraphs;
+
+ void Clear() {
+ output_args.clear();
+ inputs_and_initializers.clear();
+ outer_scope_node_args.clear();
+ node_name_to_index.clear();
+ nodes_with_subgraphs.clear();
+ }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ResolveContext);
+ };
+
+ // Initialize all the graph inputs, initializers and outputs
+ common::Status InitInputsInitializersOutputs();
+
+ // Initialize overridable initializers container
+ void ComputeOverridableInitializers();
+
+#if !defined(ORT_MINIMAL_BUILD)
+ // Build and verify node connection (edges).
+ // Verify NodeArg name/type/shape matching correctly.
+ common::Status BuildConnections(std::unordered_set<std::string>& outer_scope_node_args_consumed);
+
+ common::Status VerifyNoDuplicateName();
+
+ // Check whether <*this> graph is acyclic while performing a topological sort.
+ // Depth-first going from bottom up through the graph and checking whether there are any back edges.
+ // NodesInTopologicalOrder is updated with the nodes' indexes in topological
+ // order if <Status> returned is "OK", otherwise it's undefined.
+ common::Status PerformTopologicalSortAndCheckIsAcyclic();
+
+ common::Status PerformTypeAndShapeInferencing(const ResolveOptions& options);
+
+ // Recursively find all subgraphs including nested subgraphs
+ void FindAllSubgraphs(std::vector<Graph*>& subgraphs);
+
+ // Iterate this Graph instance and all subgraphs, calling the provided function for each.
+ common::Status ForThisAndAllSubgraphs(const std::vector<Graph*>& subgraphs, std::function<Status(Graph&)> func);
+
+ common::Status InferAndVerifyTypeMatch(Node& node, const ONNX_NAMESPACE::OpSchema& op, const ResolveOptions& options);
+
+ // perform type and shape inferencing on the subgraph and Resolve to validate
+ static common::Status InferAndVerifySubgraphTypes(const Node& node, Graph& subgraph,
+ const std::vector<const ONNX_NAMESPACE::TypeProto*>& input_types,
+ std::vector<const ONNX_NAMESPACE::TypeProto*>& output_types,
+ const Graph::ResolveOptions& options);
+
+ // Apply type-inference and type-checking to all inputs and initializers:
+ common::Status TypeCheckInputsAndInitializers();
+
+ // Compute set of input and initializer names and checking for duplicate names
+ common::Status VerifyInputAndInitializerNames();
+
+ // Infer and set type information across <*this> graph if needed, and verify type/attribute
+ // information matches between node and op.
+ common::Status VerifyNodeAndOpMatch(const ResolveOptions& options);
+
+ // Set graph inputs/outputs when resolving a graph..
+ common::Status SetGraphInputsOutputs();
+
+ // recursively accumulate and set the outer scope node args in the resolve context for all subgraphs
+ // so they can be used to resolve outer scope dependencies when running BuildConnections for the subgraphs.
+ common::Status SetOuterScopeNodeArgs(const std::unordered_set<std::string>& outer_scope_node_args);
+
+ // Clear all unused initializers
+ void CleanUnusedInitializers(const std::unordered_set<std::string>* initializer_names_to_preserve = nullptr);
+
+ std::vector<NodeArg*> CreateNodeArgs(const google::protobuf::RepeatedPtrField<std::string>& names,
+ const ArgNameToTypeMap& name_to_type_map);
+
+ void ToGraphProtoInternal(ONNX_NAMESPACE::GraphProto& graph_proto) const;
+
+ template <typename TInstance>
+ static auto GetProducerNodeImpl(
+ TInstance& instance, const std::string& node_arg_name) -> decltype(instance.GetNode(0)) {
+ auto iter = instance.node_arg_to_producer_node_.find(node_arg_name);
+ if (iter != instance.node_arg_to_producer_node_.end()) {
+ auto node_index = iter->second;
+ return instance.GetNode(node_index);
+ }
+ return nullptr;
+ }
+
+ template <typename TInstance>
+ static auto GetConsumerNodesImpl(
+ TInstance& instance, const std::string& node_arg_name) -> std::vector<decltype(instance.GetNode(0))> {
+ std::vector<decltype(instance.GetNode(0))> results;
+ auto iter = instance.node_arg_to_consumer_nodes_.find(node_arg_name);
+ if (iter != instance.node_arg_to_consumer_nodes_.end()) {
+ results.reserve(iter->second.size());
+ for (auto node_index : iter->second) {
+ results.push_back(instance.GetNode(node_index));
+ }
+ }
+ return results;
+ }
+
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
+ gsl::not_null<Node*> AllocateNode();
+
+ // Release the node.
+ // @returns false if node_index was invalid.
+ bool ReleaseNode(NodeIndex node_index);
+
+ Node& CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);
+#endif
+
+ Node* NodeAtIndexImpl(NodeIndex node_index) const {
+ // if we are trying to access a node that doesn't exist there's (most
+ // likely) either a logic issue or a graph consistency/correctness issue.
+ // use ORT_ENFORCE to prove that or uncover scenarios where we actually
+ // expect attempts to retrieve a non-existent node.
+ ORT_ENFORCE(node_index < nodes_.size(), "Validating no unexpected access using an invalid node_index. Got:",
+ node_index, " Max:", nodes_.size());
+ return nodes_[node_index].get();
+ }
+
+ const Model& owning_model_;
+
+ // GraphProto to store name, version, initializer.
+ // When serializing <*this> Graph to a GraphProto, the nodes and
+ // functions in <Graph> will also be fed into <graph_proto_> so that
+ // it's consistent with <*this> graph.
+ // This pointer is owned by parent model.
+ ONNX_NAMESPACE::GraphProto* graph_proto_;
+
+ // GraphProto that provides storage for the ONNX proto types deserialized from a flexbuffer/flatbuffer
+ ONNX_NAMESPACE::GraphProto deserialized_proto_data_;
+
+ InitializedTensorSet name_to_initial_tensor_;
+
+ std::unordered_set<std::reference_wrapper<const std::string>,
+ std::hash<std::string>, std::equal_to<std::string>>
+ sparse_tensor_names_;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry_;
+
+ std::vector<std::unique_ptr<onnxruntime::Function>> function_container_;
+#endif
+
+ // Graph nodes.
+ // Element in <nodes_> may be nullptr due to graph optimization.
+ std::vector<std::unique_ptr<Node>> nodes_;
+
+ // Wrapper of Graph nodes to provide iteration services that hide nullptr entries
+ GraphNodes iterable_nodes_{nodes_};
+
+ // Number of nodes.
+ // Normally this is smaller than the size of <m_nodes>, as some
+ // elements in <m_nodes> may be removed when doing graph optimization,
+ // or some elements may be merged, etc.
+ int num_of_nodes_ = 0;
+
+ // A flag indicates whether <*this> graph needs to be resolved.
+ bool graph_resolve_needed_ = false;
+
+ bool graph_proto_sync_needed_ = false;
+
+ // The topological order of node index used to do node and op match verification temporarily.
+ std::vector<NodeIndex> nodes_in_topological_order_;
+
+ // Full list of graph inputs. Matches number and order of inputs in the GraphProto.
+ std::vector<const NodeArg*> graph_inputs_including_initializers_;
+ bool graph_inputs_manually_set_ = false;
+
+ // Graph inputs excluding initializers.
+ std::vector<const NodeArg*> graph_inputs_excluding_initializers_;
+
+ // Overridable Initializers. The difference between graph_inputs_including_initializers_
+ // and graph_inputs_excluding_initializers_
+ std::vector<const NodeArg*> graph_overridable_initializers_;
+
+ // Graph outputs.
+ std::vector<const NodeArg*> graph_outputs_;
+ bool graph_outputs_manually_set_ = false;
+
+ // Graph value_info.
+ std::vector<const NodeArg*> value_info_;
+
+ // All node args owned by <*this> graph. Key is node arg name.
+ std::unordered_map<std::string, std::unique_ptr<NodeArg>> node_args_;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ int name_generator_ = 0;
+
+ // Strings which have been used as node names.
+ // New node name should not conflict with this set.
+ std::unordered_set<std::string> generated_node_names_;
+
+ // Strings which have been used as node_arg names.
+ // New node_arg name should not conflict this this set.
+ std::unordered_set<std::string> generated_node_arg_names_;
+
+ // node arg to its producer node
+ std::unordered_map<std::string, NodeIndex> node_arg_to_producer_node_;
+
+ // node arg to its consumer nodes
+ std::unordered_map<std::string, std::unordered_set<NodeIndex>> node_arg_to_consumer_nodes_;
+
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ const std::unordered_map<std::string, int> domain_to_version_;
+
+ // Model IR version.
+ Version ir_version_{ONNX_NAMESPACE::Version::IR_VERSION};
+
+ // Is model using latest ONNX opset
+ bool using_latest_onnx_opset_{false};
+
+ ResolveContext resolve_context_;
+
+ // the parent graph if this is a subgraph.
+ Graph* parent_graph_;
+ // the node containing the graph if parent_graph_ is not nullptr
+ const Node* parent_node_;
+
+ // NodeArgs that come from outer scope. Used when building a graph so that
+ // these don't get recorded as graph inputs in the GraphProto.
+ std::unordered_set<std::string> outer_scope_node_arg_names_;
+
+ // number of times Resolve has run.
+ int num_resolves_ = 0;
+
+ const logging::Logger& logger_;
+
+ // distinguishes between graph loaded from model file and graph created from scratch
+ const bool is_loaded_from_model_file_;
+};
+
+#if !defined(ORT_MINIMAL_BUILD)
+std::ostream& operator<<(std::ostream& out, const Graph& graph);
+#endif
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/graph_nodes.h b/include/onnxruntime/core/graph/graph_nodes.h
new file mode 100644
index 0000000..422fe95
--- /dev/null
+++ b/include/onnxruntime/core/graph/graph_nodes.h
@@ -0,0 +1,182 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+namespace onnxruntime {
+
+class Node;
+
+/**
+Class to filter out null entries from either a vector of unique_ptr<Node> or a vector of [const] Node* and
+provide an iterator interface that returns [const] Node& for the valid entries.
+*/
+template <typename TNodesContainer>
+class ValidNodes {
+ public:
+ template <typename TIterator>
+ class NodeIterator;
+
+ // optional filtering function to return a subset of nodes
+ using NodeFilterFunc = std::function<bool(NodeIndex)>;
+
+ /**
+ Construct a ValidNodes instance to provide iteration over all valid nodes in the TNodesCollection
+ @param[in] nodes Nodes to iterate, skipping invalid entries.
+ */
+ explicit ValidNodes(TNodesContainer& nodes) noexcept : nodes_(&nodes) {}
+
+ explicit ValidNodes(TNodesContainer& nodes, NodeFilterFunc&& filter_node_fn) noexcept
+ : nodes_(&nodes), filter_node_fn_{std::move(filter_node_fn)} {}
+
+ using ConstNodeIterator = NodeIterator<typename TNodesContainer::const_iterator>;
+ using MutableNodeIterator = NodeIterator<typename TNodesContainer::iterator>;
+ using ConstReverseNodeIterator = NodeIterator<typename TNodesContainer::const_reverse_iterator>;
+
+ ConstNodeIterator cbegin() const noexcept {
+ return {nodes_->cbegin(), nodes_->cend(), filter_node_fn_};
+ }
+
+ ConstNodeIterator cend() const noexcept {
+ return {nodes_->cend(), nodes_->cend(), filter_node_fn_};
+ }
+
+ ConstNodeIterator begin() const noexcept {
+ return cbegin();
+ }
+
+ ConstNodeIterator end() const noexcept {
+ return cend();
+ }
+
+ ConstReverseNodeIterator rbegin() const noexcept {
+ return {nodes_->crbegin(), nodes_->crend(), filter_node_fn_};
+ }
+
+ ConstReverseNodeIterator rend() const noexcept {
+ return {nodes_->crend(), nodes_->crend(), filter_node_fn_};
+ }
+
+ // we only allow mutable access if the container is non-const.
+ // we need to templatize the functions for enable_if to work at this level, but mandate T2 being TNodesContainer
+ template <typename T2 = TNodesContainer>
+ typename std::enable_if<!std::is_const<T2>::value, MutableNodeIterator>::type begin() noexcept {
+ static_assert(std::is_same<T2, TNodesContainer>::value, "Explicit specialization is not allowed");
+ return MutableNodeIterator(nodes_->begin(), nodes_->end(), filter_node_fn_);
+ }
+
+ template <typename T2 = TNodesContainer>
+ typename std::enable_if<!std::is_const<T2>::value, MutableNodeIterator>::type end() noexcept {
+ static_assert(std::is_same<T2, TNodesContainer>::value, "Explicit specialization is not allowed");
+ return MutableNodeIterator(nodes_->end(), nodes_->end(), filter_node_fn_);
+ }
+
+ bool empty() const noexcept { return nodes_->empty(); }
+
+ /**
+ @class NodeIterator
+ Iterator to provide const and non-const access to valid Node instances in a Graph.
+ @remarks Skips invalid nodes.
+ */
+ template <typename TIterator>
+ class NodeIterator {
+ // get the type being returned by the iterator. can't use TIterator::value_type as that is always non-const
+ using IterType = typename std::remove_reference<typename std::iterator_traits<TIterator>::reference>::type;
+ // and determine what we will return based on its constness
+ using T = typename std::conditional<std::is_const<IterType>::value,
+ const Node, // return const Node if this is a const iterator
+ Node>::type; // else return Node
+
+ public:
+ using iterator_category = std::input_iterator_tag;
+ using value_type = T;
+ using difference_type = typename TIterator::difference_type;
+ using pointer = T*;
+ using reference = T&;
+ using const_reference = const T&;
+
+ /** Construct a NodeInterator and move to the first valid node. */
+ NodeIterator<TIterator>(const TIterator current, const TIterator end, const NodeFilterFunc& filter_fn) noexcept
+ : current_{current}, end_{end}, apply_filter_{filter_fn != nullptr}, filter_func_{&filter_fn} {
+ // skip to next valid node, stopping at end if none are found
+ while (current_ < end && (*current_ == nullptr ||
+ (apply_filter_ && (*filter_func_)((*current_)->Index()) == true))) {
+ ++current_;
+ }
+ }
+
+ bool operator==(const NodeIterator<TIterator>& other) const noexcept {
+ return (current_ == other.current_);
+ }
+
+ bool operator!=(const NodeIterator<TIterator>& other) const noexcept {
+ return (current_ != other.current_);
+ }
+
+ void operator++() {
+ if (current_ < end_) {
+ while (++current_ != end_) {
+ if (*current_ != nullptr && (!apply_filter_ || (*filter_func_)((*current_)->Index()) == false))
+ break;
+ }
+ }
+ }
+
+ NodeIterator<TIterator> operator++(int) {
+ NodeIterator<TIterator> tmp{*this};
+ ++(*this);
+
+ return tmp;
+ }
+
+ /** Return the current Node&. This will be const if the iterator was returned from a const GraphNodes instance. */
+ reference operator*() {
+ // if iterator is valid we always have a non-nullptr node
+ // if this is a nullptr we're at end_ and this shouldn't be being called
+ return **current_;
+ }
+
+ pointer operator->() {
+ return current_->get();
+ }
+
+ private:
+ TIterator current_;
+ TIterator end_;
+ bool apply_filter_; // store whether filter_func_ is not nullptr and contains a callable
+ const NodeFilterFunc* filter_func_; // store as pointer so iterator is copyable
+ };
+
+ private:
+ gsl::not_null<TNodesContainer*> nodes_; // always set by ctor
+
+ // no filtering if not set. this instance owns the filter func if set.
+ NodeFilterFunc filter_node_fn_;
+};
+
+/**
+Class that provides iteration over all valid nodes in the Graph.
+*/
+class GraphNodes : public ValidNodes<std::vector<std::unique_ptr<Node>>> {
+ public:
+ GraphNodes(std::vector<std::unique_ptr<Node>>& nodes) : ValidNodes(nodes) {
+ }
+};
+
+// Variant that only ever allows const access to nodes and optionally allows filtering of the nodes.
+class ConstGraphNodes : public ValidNodes<const std::vector<std::unique_ptr<Node>>> {
+ public:
+ ConstGraphNodes(const std::vector<std::unique_ptr<Node>>& nodes) : ValidNodes(nodes) {
+ }
+
+ ConstGraphNodes(const std::vector<std::unique_ptr<Node>>& nodes,
+ GraphNodes::NodeFilterFunc&& filter_func)
+ : ValidNodes(nodes, std::move(filter_func)) {
+ }
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/graph_viewer.h b/include/onnxruntime/core/graph/graph_viewer.h
new file mode 100644
index 0000000..b2d9ee2
--- /dev/null
+++ b/include/onnxruntime/core/graph/graph_viewer.h
@@ -0,0 +1,196 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/graph/graph.h"
+#include "core/framework/session_options.h"
+#include <unordered_set>
+
+namespace onnxruntime {
+class Function;
+struct IndexedSubGraph;
+} // namespace onnxruntime
+
+namespace onnxruntime {
+
+// use value-based compare to make sure transformer output order is consistent
+struct NodeCompare {
+ bool operator()(const Node* n1, const Node* n2) const;
+};
+
+/**
+@class GraphViewer
+Class that provides a read-only view of the Graph.
+@remarks If the underlying Graph is changed, GetNodesInTopologicalOrder and GetRootNodes may become invalid.
+*/
+class GraphViewer {
+ public:
+ /**
+ Construct a GraphViewer from the provided Graph instance.
+ */
+ explicit GraphViewer(const Graph& graph);
+
+ /**
+ Construct a GraphViewer from the provided Graph instance, filtering to the nodes specified in the IndexedSubGraph
+ */
+ explicit GraphViewer(const Graph& graph, const IndexedSubGraph& filter_info);
+
+ /** Gets the Graph name. */
+ const std::string& Name() const noexcept;
+
+ /** Gets the Graph description. */
+ const std::string& Description() const noexcept;
+
+ /** Gets the path of the owning model if any **/
+ const Path& ModelPath() const noexcept { return graph_->ModelPath(); }
+
+ /**
+ Gets a tensor created from an initializer.
+ @param tensor_name The tensor name
+ @param[out] value Sets the pointer to the TensorProto if found, or nullptr if not.
+ @returns True if found. False if not.
+ */
+ bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const;
+
+ /** Returns true if an initializer value can be overridden by a graph input with the same name. */
+ bool CanOverrideInitializer() const noexcept;
+
+ /**
+ Gets the Graph inputs, excluding initializers.
+ @returns Collection of NodeArg pointers for the graph inputs, excluding inputs that have matching initializers.
+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.
+ Inputs are for filter_info_ if set.
+ */
+ const std::vector<const NodeArg*>& GetInputs() const noexcept;
+
+ /**
+ Gets the Graph inputs, including any initializers.
+ @returns Collection of NodeArg pointers for all the graph inputs.
+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.
+ Inputs are for filter_info_ if set.
+ */
+ const std::vector<const NodeArg*>& GetInputsIncludingInitializers() const noexcept;
+
+ /**
+ Gets the Graph outputs.
+ @returns Collection of NodeArg pointers for all the graph outputs.
+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.
+ Outputs are for filter_info_ if set.
+ */
+ const std::vector<const NodeArg*>& GetOutputs() const noexcept;
+
+ /** Gets all ValueInfo NodeArg instances in the Graph.
+ @remarks NOT filtered using filter_info_.
+ */
+ const std::vector<const NodeArg*>& GetValueInfo() const noexcept;
+
+ /**
+ Gets the Node instance at the specified index.
+ @param node_index Index to retrieve Node from.
+ @remarks May return nullptr if index no longer points to a valid node due to the node being freed, or if
+ node is excluded by filter_info_.
+ */
+ const Node* GetNode(NodeIndex node_index) const;
+
+ /** Gets an iterator over all the valid Nodes in the Graph.
+ @remarks Nodes are filtered using filter_info_ if set.
+ */
+ const ConstGraphNodes& Nodes() const noexcept;
+
+ /** Gets the number of valid nodes in the Graph.
+ @remarks Returns the number of nodes in filter_info_ if set.
+ */
+ int NumberOfNodes() const noexcept;
+
+ /** Gets the maximum NodeIndex value used by Nodes in the Graph. */
+ int MaxNodeIndex() const noexcept;
+
+ /** Gets the NodeIndex values for the Graph nodes, sorted into topological order.
+ @remarks Filtered using filter_info_ if set.
+ */
+ const std::vector<NodeIndex>& GetNodesInTopologicalOrder(ExecutionOrder order = ExecutionOrder::DEFAULT) const;
+
+ /**
+ Gets the NodeIndex values for the root nodes in the Graph.
+ The root nodes are the topmost nodes in the Graph that receive inputs from the Graph inputs
+ and no other nodes in the Graph.
+ @remarks Not supported if filter_info_ is set.
+ */
+ const std::vector<NodeIndex>& GetRootNodes() const;
+
+ /** Gets all tensors created from initializers. */
+ const InitializedTensorSet& GetAllInitializedTensors() const noexcept;
+
+ /**
+ Gets the NodeArg instance for the given name.
+ @returns A NodeArg if found, a nullptr if not.
+ */
+ const NodeArg* GetNodeArg(const std::string& name) const;
+
+ /** Gets the map of operator domains to their opset versions. */
+ const std::unordered_map<std::string, int>& DomainToVersionMap() const noexcept {
+ return graph_->DomainToVersionMap();
+ }
+
+ /** Checks if this is a Subgraph */
+ bool IsSubgraph() const;
+
+ /** Get the internal graph*/
+ const Graph& GetGraph() const { return *graph_; }
+
+ /**
+ returns true if 'name' is an initializer, and is constant and cannot be overridden at runtime.
+ @param check_outer_scope If true and the 'graph_' is a subgraph, check parent graph/s for 'name'
+ if the name is not found in 'graph_'.
+ */
+ bool IsConstantInitializer(const std::string& name, bool check_outer_scope) const;
+
+ /** Get the Node containing this Graph if IsSubgraph is true. Returns nullptr otherwise. */
+ const Node* ParentNode() const noexcept { return graph_->ParentNode(); }
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Get the consumer nodes of a node arg */
+ std::vector<const Node*> GetConsumerNodes(const std::string& node_arg_name) const {
+ return graph_->GetConsumerNodes(node_arg_name);
+ }
+
+ /** Get the producer node of a node arg */
+ const Node* GetProducerNode(const std::string& node_arg_name) const {
+ return graph_->GetProducerNode(node_arg_name);
+ }
+#endif
+
+ /** Get the filter info that restricts the graph viewer to a subset of nodes if set.
+ @returns Filter info or nullptr
+ */
+ const IndexedSubGraph* GetFilterInfo() const { return filter_info_; }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer);
+ GraphViewer(const Graph& graph, const IndexedSubGraph* filter_info);
+
+ const Graph* graph_;
+ ConstGraphNodes graph_nodes_;
+
+ // The NodeIndex values of the graph nodes sorted in topological order.
+ std::vector<NodeIndex> nodes_in_topological_order_;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ // The NodeIndex values of the graph nodes sorted in topological order with priority.
+ std::vector<NodeIndex> nodes_in_topological_order_with_priority_;
+#endif
+
+ // Graph root nodes.
+ std::vector<NodeIndex> root_nodes_;
+
+ // if we're limiting the view to an IndexedSubGraph we need to create a few pieces of infrastructure that would
+ // usually come from the full graph
+ const IndexedSubGraph* filter_info_{nullptr};
+ std::unordered_set<NodeIndex> filtered_node_indices_;
+ std::vector<const NodeArg*> filtered_node_inputs_;
+ std::vector<const NodeArg*> filtered_node_inputs_including_initializers_;
+ std::vector<const NodeArg*> filtered_node_outputs_;
+ InitializedTensorSet filtered_initializers_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/indexed_sub_graph.h b/include/onnxruntime/core/graph/indexed_sub_graph.h
new file mode 100644
index 0000000..b0485c4
--- /dev/null
+++ b/include/onnxruntime/core/graph/indexed_sub_graph.h
@@ -0,0 +1,61 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/graph/basic_types.h"
+#include "core/graph/onnx_protobuf.h"
+
+namespace onnxruntime {
+
+class OpKernel;
+class OpKernelInfo;
+
+/**
+@class IndexedSubGraph
+
+Class containing information about a subgraph of Nodes from a Graph.
+It contains a NodeIndex array of the Nodes covered by the subgraph,
+and the meta definition needed for representing this subgraph as a FunctionProto,
+which could be serialized/saved to a model file.
+*/
+struct IndexedSubGraph {
+ struct MetaDef {
+ std::string name; ///< Name of customized SubGraph/FunctionProto
+ std::string domain; ///< Domain of customized SubGraph/FunctionProto
+ int since_version; ///< Since version of customized SubGraph/FunctionProto.
+
+ ONNX_NAMESPACE::OperatorStatus status; ///< Status of customized SubGraph/FunctionProto.
+
+ std::vector<std::string> inputs; ///< Inputs of customized SubGraph/FunctionProto.
+ std::vector<std::string> outputs; ///< Outputs of customized SubGraph/FunctionProto.
+ NodeAttributes attributes; ///< Attributes of customized SubGraph/FunctionProto.
+
+ std::string doc_string; ///< Doc string of customized SubGraph/FunctionProto.
+ };
+
+ /** Nodes covered by this subgraph. The NodeIndex values are from the parent Graph.*/
+ std::vector<onnxruntime::NodeIndex> nodes;
+
+ /** Set the meta definition needed to represent this subgraph as a FunctionProto
+ It's needed IF AND ONLY IF there are multiple indexes contained in #nodes. */
+ void SetMetaDef(std::unique_ptr<MetaDef>&& meta_def_) {
+ meta_def = std::move(meta_def_);
+ }
+
+ /** Gets the meta definition needed to represent this subgraph as a FunctionProto.
+ @returns MetaDef instance if it has been set. nullptr if not. */
+ const MetaDef* GetMetaDef() const {
+ return meta_def.get();
+ }
+
+ private:
+ // subgraph meta definition.
+ std::unique_ptr<MetaDef> meta_def;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/node_arg.h b/include/onnxruntime/core/graph/node_arg.h
new file mode 100644
index 0000000..eeb53d2
--- /dev/null
+++ b/include/onnxruntime/core/graph/node_arg.h
@@ -0,0 +1,116 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/graph/basic_types.h"
+#include "core/graph/onnx_protobuf.h"
+
+namespace onnxruntime {
+
+// Node argument definition, for both input and output,
+// including arg name, arg type (contains both type and shape).
+//
+// Design Question: in my opinion, shape should not be part of type.
+// We may align the protobuf design with our operator registry interface,
+// which has type specified for each operator, but no shape. Well, shape
+// should be inferred with a separate shape inference function given
+// input shapes, or input tensor data sometimes.
+// With shape as part of type (current protobuf design),
+// 1) we'll have to split the "TypeProto" into type and shape in this internal
+// representation interface so that it could be easily used when doing type
+// inference and matching with operator registry.
+// 2) SetType should be always called before SetShape, otherwise, SetShape()
+// will fail. Because shape is located in a TypeProto.
+// Thoughts?
+//
+
+/**
+@class NodeArg
+Class representing a data type that is input or output for a Node, including the shape if it is a Tensor.
+*/
+class NodeArg {
+ public:
+ /**
+ Construct a new NodeArg.
+ @param name The name to use.
+ @param p_arg_type Optional TypeProto specifying type and shape information.
+ */
+ NodeArg(const std::string& name,
+ const ONNX_NAMESPACE::TypeProto* p_arg_type);
+
+ NodeArg(NodeArg&&) = default;
+ NodeArg& operator=(NodeArg&& other) = default;
+
+ /** Gets the name. */
+ const std::string& Name() const noexcept;
+
+ /** Gets the data type. */
+ ONNX_NAMESPACE::DataType Type() const noexcept;
+
+ /** Gets the TypeProto
+ @returns TypeProto if type is set. nullptr otherwise. */
+ const ONNX_NAMESPACE::TypeProto* TypeAsProto() const noexcept;
+
+ /** Gets the shape if NodeArg is for a Tensor.
+ @returns TensorShapeProto if shape is set. nullptr if there's no shape specified. */
+ const ONNX_NAMESPACE::TensorShapeProto* Shape() const;
+
+ /** Return an indicator.
+ @returns true if NodeArg is a normal tensor with a non-empty shape or a scalar with an empty shape. Otherwise, returns false. */
+ bool HasTensorOrScalarShape() const;
+
+#if !defined(ORT_MINIMAL_BUILD)
+ /** Sets the shape.
+ @remarks Shape can only be set if the TypeProto was provided to the ctor, or #SetType has been called,
+ as the shape information is stored as part of TypeProto. */
+ void SetShape(const ONNX_NAMESPACE::TensorShapeProto& shape);
+
+ /** Clears shape info.
+ @remarks If there is a mismatch during shape inferencing that can't be resolved the shape info may be removed. */
+ void ClearShape();
+
+ /** Validate and merge type [and shape] info from input_type.
+ @param strict If true, the shape update will fail if there are incompatible values.
+ If false, will be lenient and merge only shape info that can be validly processed.
+ @param override_types If true, resolve the two inputs or two outputs type when different
+ @returns Success unless there is existing type or shape info that can't be successfully updated. */
+ common::Status UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& input_type, bool strict, bool override_types, const logging::Logger& logger);
+
+ /** Validate and merge type [and shape] info from node_arg.
+ @param strict If true, the shape update will fail if there are incompatible values.
+ If false, will be lenient and merge only shape info that can be validly processed.
+ @param override_types If true, resolve the two inputs or two outputs type when different
+ @returns Success unless there is existing type or shape info that can't be successfully updated. */
+ common::Status UpdateTypeAndShape(const NodeArg& node_arg, bool strict, bool override_types, const logging::Logger& logger);
+
+#endif // !defined(ORT_MINIMAL_BUILD)
+
+ /** Gets this NodeArg as a ValueInfoProto. */
+ const NodeArgInfo& ToProto() const noexcept { return node_arg_info_; }
+
+ /** Gets a flag indicating whether this NodeArg exists or not.
+ Optional inputs are allowed in ONNX and an empty #Name represents a non-existent input argument. */
+ bool Exists() const noexcept;
+
+ private:
+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(NodeArg);
+ friend class Graph;
+
+ NodeArg(NodeArgInfo&& node_arg_info);
+
+#if !defined(ORT_MINIMAL_BUILD)
+ void SetType(ONNX_NAMESPACE::DataType p_type);
+ void SetType(const ONNX_NAMESPACE::TypeProto& type_proto);
+#endif
+
+ // Node arg PType.
+ ONNX_NAMESPACE::DataType type_;
+
+ // Node arg name, type and shape.
+ NodeArgInfo node_arg_info_;
+
+ // Flag indicates whether <*this> node arg exists or not.
+ bool exists_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/graph/onnx_protobuf.h b/include/onnxruntime/core/graph/onnx_protobuf.h
new file mode 100644
index 0000000..23a601e
--- /dev/null
+++ b/include/onnxruntime/core/graph/onnx_protobuf.h
@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+//TODO(): delete this file from public interface
+#ifdef __GNUC__
+#include "onnxruntime_config.h"
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-qualifiers"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#ifdef HAS_DEPRECATED_DECLARATIONS
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#else
+#pragma warning(push)
+#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */
+#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/
+#pragma warning(disable : 4100)
+#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/
+#pragma warning(disable : 4127)
+#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/
+#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/
+#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/
+#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/
+#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/
+#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/
+#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/
+#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/
+#pragma warning(disable : 4506) /*no definition for inline function 'function'*/
+#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/
+#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/
+#pragma warning(disable : 6011) /*Dereferencing NULL pointer*/
+#pragma warning(disable : 6387) /*'value' could be '0'*/
+#pragma warning(disable : 26495) /*Variable is uninitialized.*/
+#endif
+
+#if !defined(ORT_MINIMAL_BUILD)
+#include "onnx/defs/schema.h"
+#else
+#include "onnx/defs/data_type_utils.h"
+#endif
+
+#include "onnx/onnx_pb.h"
+#include "onnx/onnx-operators_pb.h"
+
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#else
+#pragma warning(pop)
+#endif
diff --git a/include/onnxruntime/core/graph/schema_registry.h b/include/onnxruntime/core/graph/schema_registry.h
new file mode 100644
index 0000000..0c7b0ce
--- /dev/null
+++ b/include/onnxruntime/core/graph/schema_registry.h
@@ -0,0 +1,158 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "core/graph/constants.h"
+#include "core/common/common.h"
+#include "core/common/status.h"
+#include "core/graph/onnx_protobuf.h"
+#include "core/platform/ort_mutex.h"
+#include <mutex>
+#include <deque>
+#include "sstream"
+
+namespace onnxruntime {
+using OpName_Domain_Version_Schema_Map = std::unordered_map<
+ std::string,
+ std::unordered_map<std::string, std::map<ONNX_NAMESPACE::OperatorSetVersion, ONNX_NAMESPACE::OpSchema>>>;
+
+/**
+@struct SchemaRegistryVersion
+onnxruntime schema registry is a supplement to the built-in ONNX schema.
+Every schema registry represent a collection of schema deltas from baseline_opset_version to opset_version
+*/
+struct SchemaRegistryVersion {
+ int baseline_opset_version;
+ int opset_version;
+};
+
+using DomainToVersionMap = std::unordered_map<std::string, int>;
+using DomainToVersionRangeMap = std::unordered_map<std::string, SchemaRegistryVersion>;
+
+class IOnnxRuntimeOpSchemaCollection : public ONNX_NAMESPACE::ISchemaRegistry {
+ public:
+ virtual DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const = 0;
+
+ using ISchemaRegistry::GetSchema;
+
+ const ONNX_NAMESPACE::OpSchema* GetSchema(const std::string& key, const int maxInclusiveVersion,
+ const std::string& domain) const final {
+ const ONNX_NAMESPACE::OpSchema* latest_schema = nullptr;
+ int earliest_opset_where_unchanged = std::numeric_limits<int>::max();
+ GetSchemaAndHistory(key, maxInclusiveVersion, domain, &latest_schema, &earliest_opset_where_unchanged);
+
+ assert(latest_schema == nullptr || (latest_schema->SinceVersion() <= maxInclusiveVersion &&
+ earliest_opset_where_unchanged == latest_schema->SinceVersion()));
+
+ return latest_schema;
+ }
+
+ virtual void GetSchemaAndHistory(
+ const std::string& key,
+ int maxInclusiveVersion,
+ const std::string& domain,
+ const ONNX_NAMESPACE::OpSchema** latest_schema,
+ int* earliest_opset_where_unchanged) const = 0;
+};
+
+/**
+@class OnnxRuntimeOpSchemaRegistry
+
+OnnxRuntimeOpSchemaRegistry is used to provide supplement for built-in ONNX schemas.
+Each OnnxRuntimeOpSchemaRegistry must register complete opsets delta from a baseline version to max opset version.
+(Please notice that baseline opsets are not include in the delta)
+
+For example, ONNXRuntime is build with ONNX 1.2 which is at opset7, to use ONNX opset8 and opset9,
+user could create a OnnxRuntimeOpSchemaRegistry and config it as {baseline_opset_version = 7, opset_version = 9}
+it means this OnnxRuntimeOpSchemaRegistry contains the complete delta from opset7 to opset9.
+*/
+class OnnxRuntimeOpSchemaRegistry : public IOnnxRuntimeOpSchemaCollection {
+ public:
+ OnnxRuntimeOpSchemaRegistry() = default;
+
+ common::Status SetBaselineAndOpsetVersionForDomain(
+ const std::string& domain,
+ int baseline_opset_version,
+ int opset_version);
+
+ DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const override;
+
+ // OnnxRuntimeOpSchemaRegistry must register complete delta for a opset.
+ common::Status RegisterOpSet(
+ std::vector<ONNX_NAMESPACE::OpSchema>& schemas,
+ const std::string& domain,
+ int baseline_opset_version,
+ int opset_version);
+
+ using IOnnxRuntimeOpSchemaCollection::GetSchema;
+
+ void GetSchemaAndHistory(const std::string& key, int maxInclusiveVersion, const std::string& domain,
+ const ONNX_NAMESPACE::OpSchema** latest_schema,
+ int* earliest_opset_where_unchanged) const override;
+
+ bool empty() const {
+ return map_.empty();
+ }
+
+ private:
+ common::Status RegisterOpSchema(ONNX_NAMESPACE::OpSchema&& op_schema);
+
+ common::Status RegisterOpSchemaInternal(ONNX_NAMESPACE::OpSchema&& op_schema);
+
+ OrtMutex mutex_;
+
+ OpName_Domain_Version_Schema_Map map_;
+ DomainToVersionRangeMap domain_version_range_map_;
+};
+
+/**
+@class SchemaRegistryManager
+
+SchemaRegistryManager provides a view based on built-in ONNX schema and a list of
+OnnxRuntimeOpSchemaRegistry as supplement.
+
+The user needs to make sure the customized schema registry is valid, otherwise the behavior is undefined.
+
+@todo We may add more consistency checks later.
+*/
+class SchemaRegistryManager : public onnxruntime::IOnnxRuntimeOpSchemaCollection {
+ public:
+ /**
+ Register a new schema registry instance.
+ @remarks The schema registry priority is the reverse of registration order. i.e. the last registry added will be
+ searched first for a matching OpSchema.
+ */
+ void RegisterRegistry(std::shared_ptr<IOnnxRuntimeOpSchemaCollection> registry);
+
+ /** Gets the latest opset versions.
+ @param is_onnx_only If true, return the latest ONNX schemas. If false, return the latest schemas for all domains.
+ */
+ DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const override;
+
+ /** Gets the last released opset versions.
+ @param is_onnx_only If true, return ONNX schemas only. If false, return the schemas for all domains.
+ */
+ DomainToVersionMap GetLastReleasedOpsetVersions(bool is_onnx_only) const ;
+ /**
+ Gets the OpSchema and its history.
+ Searches custom schema registries starting with the last one added. \
+ If the OpSchema is not found the default ONNX schema registry is searched.
+
+ @param key Operator type.
+ @param max_inclusive_version Maximum opset version allowed, inclusive.
+ @param domain The domain of the operator.
+ @param[out] latest_schema Returns the latest OpSchema if found. nullptr otherwise.
+ @param[out] earliest_opset_where_unchanged The earliest opset version preceding max_inclusive_version where the
+ operator is known to be unchanged.
+ */
+ void GetSchemaAndHistory(const std::string& key, int max_inclusive_version, const std::string& domain,
+ const ONNX_NAMESPACE::OpSchema** latest_schema,
+ int* earliest_opset_where_unchanged) const override;
+
+ private:
+ void GetDomainToVersionMapForRegistries(DomainToVersionMap& domain_version_map, bool is_onnx_only) const;
+
+ std::deque<std::shared_ptr<IOnnxRuntimeOpSchemaCollection>> registries;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/optimizer/graph_transformer.h b/include/onnxruntime/core/optimizer/graph_transformer.h
new file mode 100644
index 0000000..865a9f3
--- /dev/null
+++ b/include/onnxruntime/core/optimizer/graph_transformer.h
@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <string>
+#include <unordered_set>
+
+#include "core/common/common.h"
+#include "core/graph/graph_viewer.h"
+#include "core/optimizer/graph_transformer_level.h"
+
+namespace onnxruntime {
+
+/**
+@class GraphTransformer
+
+The interface for in-place transformation of a Graph.
+*/
+class GraphTransformer {
+ public:
+ GraphTransformer(const std::string& name, const std::unordered_set<std::string>& compatible_execution_providers = {})
+ : name_(name), compatible_provider_types_(compatible_execution_providers) {
+ }
+
+ virtual ~GraphTransformer() = default;
+
+ /** Gets the name of this graph transformer. */
+ const std::string& Name() const noexcept {
+ return name_;
+ }
+
+ const std::unordered_set<std::string>& GetCompatibleExecutionProviders() const noexcept {
+ return compatible_provider_types_;
+ }
+
+ /** Apply the in-place transformation defined by this transformer to the provided Graph instance.
+ @param[out] modified Set to true if the Graph was modified.
+ @returns Status with success or error information.
+ */
+ common::Status Apply(Graph& graph, bool& modified, const logging::Logger& logger) const;
+
+ virtual bool ShouldOnlyApplyOnce() const { return false; }
+
+ protected:
+ /** Helper method to call ApplyImpl on any subgraphs in the Node. */
+ common::Status Recurse(Node& node, bool& modified, int graph_level, const logging::Logger& logger) const {
+ int subgraph_level = ++graph_level;
+ for (auto& entry : node.GetAttributeNameToMutableSubgraphMap()) {
+ auto& subgraph = *entry.second;
+ ORT_RETURN_IF_ERROR(ApplyImpl(subgraph, modified, subgraph_level, logger));
+ }
+
+ return Status::OK();
+ }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformer);
+
+ // Apply the transform to the graph.
+ // graph_level is 0 for the main graph, and is incremented when descending into the subgraph of a node.
+ // You MUST call Recurse for all valid Nodes in the graph to ensure any subgraphs in control flow nodes
+ // (Scan/If/Loop) are processed as well.
+ // You should avoid calling Graph::Resolve in ApplyImpl unless you are 100% sure it's required. In most cases
+ // the call to Graph::Resolve in Apply prior to ApplyImpl being called, and after ApplyImpl fore the main graph
+ // completes (if 'modified' is true) should suffice.
+ virtual common::Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger)
+ const = 0;
+
+ const std::string name_;
+ const std::unordered_set<std::string> compatible_provider_types_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/optimizer/graph_transformer_level.h b/include/onnxruntime/core/optimizer/graph_transformer_level.h
new file mode 100644
index 0000000..cc05b21
--- /dev/null
+++ b/include/onnxruntime/core/optimizer/graph_transformer_level.h
@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/common.h"
+
+namespace onnxruntime {
+
+// Graph transformer level
+// refer to docs/ONNX_Runtime_Graph_Optimizations.md for details
+enum class TransformerLevel : int {
+ Default = 0, // required transformers only
+ Level1, // basic optimizations
+ Level2, // extended optimizations
+ Level3, // layout optimizations
+ // The max level should always be same as the last level.
+ MaxLevel = Level3
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/optimizer/graph_transformer_utils.h b/include/onnxruntime/core/optimizer/graph_transformer_utils.h
new file mode 100644
index 0000000..b5d0287
--- /dev/null
+++ b/include/onnxruntime/core/optimizer/graph_transformer_utils.h
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <gsl/gsl>
+
+#include "core/optimizer/graph_transformer.h"
+#include "core/optimizer/rule_based_graph_transformer.h"
+#include "core/optimizer/rewrite_rule.h"
+
+namespace onnxruntime {
+struct FreeDimensionOverride;
+class IExecutionProvider;
+
+namespace optimizer_utils {
+
+/** Generates all predefined rules for this level.
+ If rules_to_enable is not empty, it returns the intersection of predefined rules and rules_to_enable.
+ TODO: This is visible for testing at the moment, but we should rather make it private. */
+std::vector<std::unique_ptr<RewriteRule>> GenerateRewriteRules(TransformerLevel level,
+ const std::vector<std::string>& rules_to_enable = {});
+
+/** Generates all predefined (both rule-based and non-rule-based) transformers for this level.
+ If transformers_and_rules_to_enable is not empty, it returns the intersection between the predefined transformers/rules
+ and the transformers_and_rules_to_enable. */
+std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerLevel level,
+ gsl::span<const FreeDimensionOverride> free_dimension_overrides,
+ const IExecutionProvider& execution_provider /*required by constant folding*/,
+ const std::vector<std::string>& rules_and_transformers_to_enable = {});
+
+/** Given a TransformerLevel, this method generates a name for the rule-based graph transformer of that level. */
+std::string GenerateRuleBasedTransformerName(TransformerLevel level);
+
+/** Generates all rule-based transformers for this level. */
+std::unique_ptr<RuleBasedGraphTransformer> GenerateRuleBasedGraphTransformer(TransformerLevel level,
+ const std::vector<std::string>& rules_to_enable,
+ const std::unordered_set<std::string>& compatible_execution_providers);
+
+} // namespace optimizer_utils
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/optimizer/rewrite_rule.h b/include/onnxruntime/core/optimizer/rewrite_rule.h
new file mode 100644
index 0000000..43e7489
--- /dev/null
+++ b/include/onnxruntime/core/optimizer/rewrite_rule.h
@@ -0,0 +1,89 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/common.h"
+#include "core/graph/graph_viewer.h"
+
+namespace onnxruntime {
+
+/**
+@class RewriteRule
+
+The base class for a rewrite rule. A rewrite rule represents a semantics-preserving transformation of a
+computation graph. It can be used to represent, for example, the elimination of operators that serve as
+no-ops (e.g., dropout during inference), as well as inlining of "function" definitions or the dual operation
+of replacing a complex expression by an equivalent function-call). Unlike the more general GraphTransformer,
+a rewrite rule is a more local transformation that is triggered on a particular node of the graph.
+
+Each rule has a set of conditions and a body. The conditions have to be satisfied for the body of the rule
+to be triggered. Therefore, when creating a new rewrite rule, two main functions have to be implemented:
+- SatisfyCondition defines the condition checks. It is advisable to add the more selective checks first,
+ because those will lead to discarding fast rules that cannot be applied on a node.
+- Apply is the actual body of the rule that will be executed if SatisfyCondition returns true for a particular
+ node. Note that additional, more complex checks can be included in the Apply if putting them in the
+ SatisfyCondition would lead to duplicate work (e.g., when we make a check on a Node attribute but we need
+ that attribute to execute the rule too).
+In general, simple fast checks are a better fit for SatisfyCondition, whereas more complex ones can be added
+in the Apply.
+
+In order to avoid evaluating the SatisfyCondition for each rule and each node of the graph, each rewrite rule
+should specify the target op types for which a rule will be evaluated, by overriding the TargetOpTypes() function.
+If the op type of a node is not included in the target op types of a rule, that rule would not be considered at all.
+If the list of op types is left empty, that rule will be triggered for every op type.
+*/
+class RewriteRule {
+ public:
+ /**
+ @class RewriteRuleEffect
+
+ Class used to indicate the effect of rule application on a graph's node.
+ */
+ enum class RewriteRuleEffect : uint8_t {
+ kNone, // The rewrite rule has not modified the graph.
+ kUpdatedCurrentNode, // The rewrite rule updated (but did not remove) the node on which it was triggered.
+ kRemovedCurrentNode, // The rewrite rule removed the node on which it was triggered.
+ kModifiedRestOfGraph // The rewrite rule modified nodes other than the one it was triggered on.
+ };
+
+ RewriteRule(const std::string& name) : name_(name) {}
+
+ virtual ~RewriteRule() = default;
+
+ /** Gets the name of this rewrite rule. */
+ const std::string& Name() const noexcept {
+ return name_;
+ }
+
+ /** Returns the node op types for which this rule will be triggered. If the op type of a node is not included in the
+ target op types of a rule, that rule would not be considered at all. Returning an empty list indicates that we
+ will attempt to trigger the rule for every op type. */
+ virtual std::vector<std::string> TargetOpTypes() const noexcept = 0;
+
+ /** Checks if the condition of the rule is satisfied, and if so applies the body of the rule.
+ @param[in] graph The Graph.
+ @param[in] node The Node to apply the rewrite to.
+ @param[out] rule_effect Enum to indicate if and how the graph was modified as a result of the rule application.
+ @returns Status indicating success or providing error information */
+ common::Status CheckConditionAndApply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger& logger) const {
+ return SatisfyCondition(graph, node, logger) ? Apply(graph, node, rule_effect, logger) : Status::OK();
+ }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RewriteRule);
+
+ const std::string name_;
+
+ /** Checks if the Node of the given Graph satisfies the conditions of this rule. The body of the rule will be
+ evaluated if this condition function returns true. This can include a more complex pattern matching (conditions
+ on the ascending or descending nodes of the node for which this rule was triggered) or some other properties
+ of the nodes. */
+ virtual bool SatisfyCondition(const Graph& graph, const Node& node, const logging::Logger& logger) const = 0;
+
+ /** This is the actual body of the rule that performs the graph transformation. The transformation happens in-place.
+ The return-value of node may be different from the input-value due to rewriting.
+ The value of "rule_effect" indicates whether and how the graph was modified by the rule. */
+ virtual common::Status Apply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger& logger) const = 0;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/optimizer/rule_based_graph_transformer.h b/include/onnxruntime/core/optimizer/rule_based_graph_transformer.h
new file mode 100644
index 0000000..04b5072
--- /dev/null
+++ b/include/onnxruntime/core/optimizer/rule_based_graph_transformer.h
@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include "core/common/common.h"
+#include "core/graph/graph_viewer.h"
+#include "core/optimizer/graph_transformer.h"
+#include "core/optimizer/rewrite_rule.h"
+
+namespace onnxruntime {
+
+/**
+@class RuleBasedGraphTransformer
+
+Rule-based graph transformer that provides an API to register rewrite rules
+and an API to apply all applicable rules to a Graph.
+
+Represents an IGraphTransformer determined by a set of rewrite rules.
+The transformer will apply all the rewrite rules iteratively as determined by the underlying rewriting strategy.
+Several rewriting-strategies are possible when traversing the graph and applying rewrite rules,
+each with different trade offs. At the moment, we define one that performs top-down traversal of nodes.
+
+@TODO: Is a bottom-up traversal more efficient?
+@TODO: Is it worth adding the max number of passes a rule should be applied for?
+@TODO: We need to define a contract about whether a rewrite rule is allowed to leave
+ the graph in an inconsistent state (this will determine when and where we will be
+ calling Graph::resolve().
+*/
+class RuleBasedGraphTransformer : public GraphTransformer {
+ public:
+ RuleBasedGraphTransformer(const std::string& name,
+ const std::unordered_set<std::string>& compatible_execution_providers = {})
+ : GraphTransformer(name, compatible_execution_providers) {}
+
+ /** Registers a rewrite rule in this transformer. */
+ Status Register(std::unique_ptr<RewriteRule> rule);
+
+ /** Gets the list of registered rewrite rules that will be triggered on nodes with the given op type
+ by this rule-based transformer.
+ @returns a pointer to the vector containing all the registered rewrite rules. */
+ const std::vector<std::reference_wrapper<const RewriteRule>>* GetRewriteRulesForOpType(const std::string& op_type) const {
+ auto rules = op_type_to_rules_.find(op_type);
+ return (rules != op_type_to_rules_.cend()) ? &rules->second : nullptr;
+ }
+
+ /** Gets the rewrite rules that are evaluated on all nodes irrespective of their op type.
+ @returns a pointer to the vector containing all such rewrite rules or nullptr if no such rule. */
+ const std::vector<std::reference_wrapper<const RewriteRule>>* GetAnyOpRewriteRules() const {
+ return &any_op_type_rules_;
+ }
+
+ /** Returns the total number of rules that are registered in this transformer. */
+ size_t RulesCount() const;
+
+ protected:
+ /** Applies the given set of rewrite rules on the Node of this Graph.
+ @param[in] graph The Graph.
+ @param[in] node The Node to apply the rules to.
+ @param[in] rules The vector of RewriteRules that will be applied to the Node.
+ @param[out] rule_effect Enum that indicates whether and how the graph was modified as a result of
+ applying rules on this node.
+ @returns Status indicating success or providing error information. */
+ common::Status ApplyRulesOnNode(Graph& graph, Node& node,
+ const std::vector<std::reference_wrapper<const RewriteRule>>& rules,
+ RewriteRule::RewriteRuleEffect& rule_effect, const logging::Logger& logger) const;
+
+ private:
+ using RuleEffect = RewriteRule::RewriteRuleEffect;
+
+ // The list of unique pointers for all rules (so that rules can be registered for several op types).
+ std::vector<std::unique_ptr<RewriteRule>> rules_;
+ // Map that associates a node's op type with the vector of rules that are registered to be triggered for that node.
+ std::unordered_map<std::string, std::vector<std::reference_wrapper<const RewriteRule>>> op_type_to_rules_;
+ // Rules that will be evaluated regardless of the op type of the node.
+ std::vector<std::reference_wrapper<const RewriteRule>> any_op_type_rules_;
+
+ // Performs a single top-down traversal of the graph and applies all registered rules.
+ common::Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
+};
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/platform/Barrier.h b/include/onnxruntime/core/platform/Barrier.h
new file mode 100644
index 0000000..915cfc5
--- /dev/null
+++ b/include/onnxruntime/core/platform/Barrier.h
@@ -0,0 +1,81 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2016 Dmitry Vyukov <[email protected]>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <assert.h>
+
+#include "core/common/spin_pause.h"
+#include "core/platform/ort_mutex.h"
+
+#include <mutex>
+#include <atomic>
+
+namespace onnxruntime {
+class Barrier {
+ public:
+ explicit Barrier(unsigned int count, bool spin = false)
+ : state_(count << 1), notified_(false), spin_(spin) {
+ assert(((count << 1) >> 1) == count);
+ }
+#ifdef NDEBUG
+ ~Barrier() = default;
+#else
+ ~Barrier() {
+ assert((state_ >> 1) == 0);
+ }
+#endif
+
+ void Notify(unsigned int c = 1) {
+ unsigned int delta = c << 1;
+ unsigned int v = state_.fetch_sub(delta, std::memory_order_acq_rel) - delta;
+ if (v != 1) {
+ // Clear the lowest bit (waiter flag) and check that the original state
+ // value was not zero. If it was zero, it means that notify was called
+ // more times than the original count.
+ assert(((v + delta) & ~1) != 0);
+ return; // either count has not dropped to 0, or waiter is not waiting
+ }
+ std::unique_lock<OrtMutex> l(mu_);
+ assert(!notified_);
+ notified_ = true;
+ cv_.notify_all();
+ }
+
+ void Wait() {
+ if (spin_) {
+ while ((state_ >> 1) != 0) {
+ onnxruntime::concurrency::SpinPause();
+ }
+ } else {
+ unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel);
+ if ((v >> 1) == 0)
+ return;
+ std::unique_lock<OrtMutex> l(mu_);
+ while (!notified_) {
+ cv_.wait(l);
+ }
+ }
+ }
+
+ private:
+ OrtMutex mu_;
+ OrtCondVar cv_;
+ std::atomic<unsigned int> state_; // low bit is waiter flag
+ bool notified_;
+ const bool spin_;
+};
+
+// Notification is an object that allows a user to to wait for another
+// thread to signal a notification that an event has occurred.
+//
+// Multiple threads can wait on the same Notification object,
+// but only one caller must call Notify() on the object.
+struct Notification : Barrier {
+ Notification() : Barrier(1){};
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h
new file mode 100644
index 0000000..f24a114
--- /dev/null
+++ b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h
@@ -0,0 +1,1391 @@
+// This file is part of Eigen, a lightweight C++ template library
+// for linear algebra.
+//
+// Copyright (C) 2016 Dmitry Vyukov <[email protected]>
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+/* Modifications Copyright (c) Microsoft. */
+
+#include <type_traits>
+
+#pragma once
+#include "onnxruntime_config.h"
+// build/external/eigen/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h:162:71:
+// error: ignoring attributes on template argument "Eigen::PacketType<const float, Eigen::DefaultDevice>::type {aka
+// __vector(4) float}" [-Werror=ignored-attributes]
+#if defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 4127)
+#pragma warning(disable : 4805)
+#endif
+
+#include "unsupported/Eigen/CXX11/ThreadPool"
+
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#include "core/common/denormal.h"
+#include "core/common/make_unique.h"
+#include "core/common/spin_pause.h"
+#include "core/platform/ort_mutex.h"
+#include "core/platform/Barrier.h"
+
+// ORT thread pool overview
+// ------------------------
+//
+// The ORT thread pool implementation is split into two layers. This
+// file provides the low-level component. See the accompanying
+// comments in threadpool.h for the high-level component.
+//
+// The code here is derived from the Eigen non-blocking thread pool,
+// although many parts have been updated over time. The main
+// abstractions used here are:
+//
+// - The thread pool maintains a set of OS threads running
+// ThreadPoolTempl::WorkerLoop.
+//
+// Each thread has its own RunQueue object, holding a queue of tasks
+// that have been pushed to the thread for execution. The main work
+// loop is to pop a task from the head of the queue, and to execute
+// it to completion. If the worker's run queue is empty then it
+// will spin waiting for work, then attempt to steal tasks from
+// other threads' queues, and then block in the OS if it cannot find
+// work.
+//
+// This spin-then-block behavior is configured via a flag provided
+// when creating the thread pool, and by the constant spin_count.
+//
+// - Although all tasks are simple void()->void functions,
+// conceptually there are three different kinds:
+//
+// - One-shot tasks submitted externally via the Schedule() method.
+// These tasks are used to support asynchronous work. These are
+// used in the parallel executor, but otherwise are not widely
+// used outside of test harnesses (see threadpool_test.cc for some
+// examples).
+//
+// - Tasks for running a parallel loop.
+//
+// The tasks themselves are defined in threadpool.cc, and are
+// submitted to the run queues via RunInParallel->SummonWorkers.
+// Each task will loop internally, picking off iterations from the
+// user's code via atoic-fetch-and-add, until the loop is
+// complete.
+//
+// This two-layer approach lets us separate out the
+// super-lightweight per-iteration-batch work from the more
+// costsly per-loop work of managing Task objects.
+//
+// - Tasks for running a parallel section. This is an extension of
+// the approach taken for parallel loops. However, the Tasks are
+// defined in this file, and can pick up iterations from a series
+// of different parallel loops. The tasks are defined in
+// RunInParallelSection->SummonWorkers.
+//
+// The additional layer of parallel sections is a further way to
+// amortize costs: the work done creating the tasks can be
+// performed once, and then exploited over a series of loops.
+//
+// There are a few aspects of the modified Eigen thread pool to
+// highlight:
+//
+// - The run queues follow the usual approach of having push/pop
+// operations on the front/back, and optimizing the PopFront case
+// for single-threaded use by the thread owning the run queue.
+//
+// However, we support an additional Revoke operation to replace an
+// item in the middle of a queue with a tombstone. This operation
+// is used at the end of parallel loops and parallel sections to
+// remove any tasks that were created but not yet executed. Once
+// revoked, a thread can rely on the fact that the task will no
+// longer execute. Revocation helps manage captured state in
+// parallel loops: the alternatives would be (i) waiting for all
+// tasks that captured state to reach the head of their queues and
+// execute, or (ii) use heap-allocated state in tasks, and use a
+// technique such as reference counting to de-allocate it.
+//
+// To support revoation, each thread has a unique "Tag" to identify
+// the items that it adds to the work queues. A thread can revoke
+// an item only if it has the thread's own tag.
+//
+// - The worker threads maintain a best-effort bitmap in
+// good_worker_hints_ of which threads to push work to. A thread
+// controls its status via SetGoodWorkerHint. A thread is a "good"
+// worker when it is actively spinning for work, meaning both that
+// it is not blocked in the OS, and that it is not busy with work
+// already.
+//
+// This heuristic aims to avoid waking additional sleeping threads
+// where possible, and in a series of parallel loops or parallel
+// sections to push the work to the same set of threads each time.
+
+namespace onnxruntime {
+namespace concurrency {
+
+class ThreadPoolParallelSection;
+class ThreadPoolLoop;
+
+// Align to avoid false sharing with prior fields. If required,
+// alignment or padding must be added subsequently to avoid false
+// sharing with later fields. Note that:
+//
+// - The __x86_64__ value is twice the line size (64 bytes). This
+// accounts for 2-line prefetch behavior on some cores.
+//
+// - Ideally, ORT_ALIGN_TO_AVOID_FALSE_SHARING is used. However, the
+// definition of ThreadPoolParallelSection uses naive padding
+// because C++11 does not support alignment constraints on
+// allocation or expose stdlib.h aligned_alloc. C++17 introduces
+// support for aligned allocation which we could use here.
+
+#if defined(__x86_64__)
+#define ORT_FALSE_SHARING_BYTES 128
+#else
+#define ORT_FALSE_SHARING_BYTES 64
+#endif
+
+#define ORT_ALIGN_TO_AVOID_FALSE_SHARING alignas(ORT_FALSE_SHARING_BYTES)
+
+struct PaddingToAvoidFalseSharing {
+ char padding[ORT_FALSE_SHARING_BYTES];
+};
+
+// Extended Eigen thread pool interface, avoiding the need to modify
+// the ThreadPoolInterface.h header from the external Eigen
+// repository.
+
+class ExtendedThreadPoolInterface : public Eigen::ThreadPoolInterface {
+ public:
+ // Start/end a parallel section, within which calls to
+ // RunInParallelSection may be made. Parallel sections are
+ // non-nesting.
+ virtual std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)> AllocateParallelSection() = 0;
+ virtual void StartParallelSection(ThreadPoolParallelSection &ps) = 0;
+ virtual void EndParallelSection(ThreadPoolParallelSection &ps) = 0;
+
+ // Run fn with up to n degree-of-parallelism enlisting the thread
+ // pool for help. The degree-of-parallelism includes the caller,
+ // and so if n==1 then the function will run directly in the caller.
+ //
+ // The fork-join synchronization is handled in the thread pool, and
+ // so any state captured by fn() is safe from concurrent access once
+ // RunInParallelSection returns.
+ //
+ // The parameter idx provides a loop-local thread ID in the range
+ // [0,k) where k<=n.
+ virtual void RunInParallelSection(ThreadPoolParallelSection &ps,
+ std::function<void(unsigned idx)> fn,
+ unsigned n) = 0;
+
+ // Special case alternative to RunInParallelSection for use without
+ // an existing parallel section. Ideally we would use a single
+ // iplemenation and a stack-allocated ThreadPoolParallelSection.
+ //
+ // However, on the BM_ThreadPoolParallelFor microbenchmark I saw
+ // ~20% overhead on the resulting single-loop parallel sections.
+ // There are some additional costs (~5%) for additional invocations
+ // through lambda functions on loop entry. Most significantly, on
+ // loop exit, we incurred ~15% cost by no longer being able to
+ // overlap clean-up of unused Task objects in EndParallelSection
+ // with waiting for loop iterations to complete.
+ //
+ // [ Note that this 20% overhead is more than paid for when we have
+ // two loops execute in series in a parallel section. ]
+ virtual void RunInParallel(std::function<void(unsigned idx)> fn,
+ unsigned n) = 0;
+};
+
+
+class ThreadPoolParallelSection {
+ public:
+ // State accessed only by the main thread
+ // --------------------------------------
+
+ // Tasks successfully submitted to the work queues. This sets the
+ // maximum degree of parallelism that the section will support.
+ std::vector<std::pair<int,unsigned>> tasks;
+
+ // State shared between the main thread and worker threads
+ // -------------------------------------------------------
+
+ // Flag to signal termination of the parallel section
+ std::atomic<bool> active{false};
+
+ std::atomic<unsigned> worker_idx{0};
+
+ // Count of the number of tasks that completed normally. Other
+ // tasks may be running currently, or may be present in work queues,
+ // or may have been removed from the queues by
+ // RunQueue::RevokeWithTag.
+ PaddingToAvoidFalseSharing padding_1;
+ std::atomic<unsigned> tasks_finished{0};
+ PaddingToAvoidFalseSharing padding_2;
+
+ // If non-null, the current loop that tasks should be executing. We
+ // need to be careful on access to the contents of current_loop
+ // because it can be stack allocated on the thread entering the
+ // loop:
+ //
+ // - Readers increment workers_in_loop and then read current_loop
+ //
+ // - Writers wishing to deallocate *current_loop must first clear
+ // current_loop and then wait for workers_in_loop==0
+ std::atomic<ThreadPoolLoop *> current_loop{nullptr};
+ std::atomic<unsigned> workers_in_loop{0};
+};
+
+class ThreadPoolLoop {
+ public:
+ ThreadPoolLoop(std::function<void(unsigned)> f, unsigned t) : fn(std::move(f)), threads_needed(t) {
+ }
+
+ const std::function<void(unsigned)> fn;
+ const unsigned threads_needed;
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ThreadPoolLoop);
+};
+
+template <typename Work, typename Tag, unsigned kSize>
+class RunQueue {
+ public:
+ RunQueue() : front_(0), back_(0) {
+ // require power-of-two for fast masking
+ assert((kSize & (kSize - 1)) == 0);
+ assert(kSize > 2); // why would you do this?
+ assert(kSize <= (64 << 10)); // leave enough space for counter
+ for (unsigned i = 0; i < kSize; i++) array_[i].state.store(ElemState::kEmpty, std::memory_order_relaxed);
+ }
+
+ ~RunQueue() {
+ assert(Size() == 0);
+ }
+
+ // PushFront inserts w at the beginning of the queue.
+ // If queue is full returns w, otherwise returns default-constructed Work.
+ Work PushFront(Work w) {
+ unsigned front = front_.load(std::memory_order_relaxed);
+ Elem& e = array_[front & kMask];
+ ElemState s = e.state.load(std::memory_order_relaxed);
+ if (s != ElemState::kEmpty ||
+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))
+ return w;
+ front_.store(front + 1 + (kSize << 1), std::memory_order_relaxed);
+ e.w = std::move(w);
+ e.tag = Tag();
+ e.state.store(ElemState::kReady, std::memory_order_release);
+ return Work();
+ }
+
+ // PopFront removes and returns the first element in the queue.
+ // If the queue was empty returns default-constructed Work.
+ Work PopFront() {
+ unsigned front;
+ Elem *e;
+ ElemState s;
+
+ // Drain revoked items from the front of the queue. CAS to busy to synchronize with
+ // any attempt to take the same item from the back of the queue.
+ do {
+ front = front_.load(std::memory_order_relaxed);
+ e = &array_[(front - 1) & kMask];
+ s = e->state.load(std::memory_order_relaxed);
+ if (s == ElemState::kRevoked &&
+ e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {
+ e->state.store(ElemState::kEmpty, std::memory_order_release);
+ front = ((front - 1) & kMask2) | (front & ~kMask2);
+ front_.store(front, std::memory_order_relaxed);
+ }
+ } while (s == ElemState::kRevoked);
+
+ // Attempt to take next item. State kEmpty shows the queue is empty, kBusy shows
+ // the work is in progress on the item at the front of the queue.
+ if (s != ElemState::kReady ||
+ !e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))
+ return Work();
+ Work w = std::move(e->w);
+ e->tag = Tag();
+ e->state.store(ElemState::kEmpty, std::memory_order_release);
+ front = ((front - 1) & kMask2) | (front & ~kMask2);
+ front_.store(front, std::memory_order_relaxed);
+ return w;
+ }
+
+ // PushBack adds w at the end of the queue.
+ // If queue is full returns w, otherwise returns default-constructed Work.
+ Work PushBack(Work w) {
+ std::unique_lock<OrtMutex> lock(mutex_);
+ unsigned back = back_.load(std::memory_order_relaxed);
+ Elem& e = array_[(back - 1) & kMask];
+ ElemState s = e.state.load(std::memory_order_relaxed);
+ if (s != ElemState::kEmpty ||
+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))
+ return w;
+ back = ((back - 1) & kMask2) | (back & ~kMask2);
+ back_.store(back, std::memory_order_relaxed);
+ e.w = std::move(w);
+ e.tag = Tag();
+ e.state.store(ElemState::kReady, std::memory_order_release);
+ return Work();
+ }
+
+ // PushBackWithTag adds w at the end of the queue. The tag value can be used on a
+ // subsequent call to RevokeWithTag to remove the item from the queue in combination
+ // with w_idx. Typically the tag will be a per-thread ID to distinguish work
+ // submitted from different threads.
+ //
+ // If the queue is full, returns w, otherwise returns default-constructed work.
+ Work PushBackWithTag(Work w, Tag tag, unsigned &w_idx) {
+ std::unique_lock<OrtMutex> lock(mutex_);
+ unsigned back = back_.load(std::memory_order_relaxed);
+ w_idx = (back-1) & kMask;
+ Elem& e = array_[w_idx];
+ ElemState s = e.state.load(std::memory_order_relaxed);
+ if (s != ElemState::kEmpty ||
+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))
+ return w;
+ back = ((back - 1) & kMask2) | (back & ~kMask2);
+ back_.store(back, std::memory_order_relaxed);
+ e.w = std::move(w);
+ e.tag = tag;
+ e.state.store(ElemState::kReady, std::memory_order_release);
+ return Work();
+ }
+
+ // PopBack removes and returns the last elements in the queue.
+ Work PopBack() {
+ if (Empty())
+ return Work();
+ std::unique_lock<OrtMutex> lock(mutex_);
+ unsigned back;
+ Elem *e;
+ ElemState s;
+
+ // Drain revoked items from the back of the queue. CAS to busy to synchronize with
+ // any attempt to take the same item from the front of the queue.
+ do {
+ back = back_.load(std::memory_order_relaxed);
+ e = &array_[back & kMask];
+ s = e->state.load(std::memory_order_relaxed);
+ if (s == ElemState::kRevoked &&
+ e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {
+ e->state.store(ElemState::kEmpty, std::memory_order_release);
+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);
+ }
+ } while (s == ElemState::kRevoked);
+
+ if (s != ElemState::kReady ||
+ !e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))
+ return Work();
+ Work w = std::move(e->w);
+ e->tag = Tag();
+ e->state.store(ElemState::kEmpty, std::memory_order_release);
+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);
+ return w;
+ }
+
+ // RevokeItem removes a work item from the queue. Items are identified positionally,
+ // and so a tag is used to detect whether the same position is occupied by a
+ // different work item at the time of removal. RevokeWithTags lets threads offer work
+ // for parallel execution, and then revoke the offer prior to the work executing (for
+ // instance if the thread itself completes all of the work). Revoking the work
+ // lets the thread deallocate state that might otherwise have been captured by the work item
+ // and accessed by it.
+ //
+ // Return true iff the item is successfully revoked. If the item is not revoked then
+ // the caller must assume that it may still execute, for instance because it
+ // has been pop'd from the queue concurrent with the revocation request.
+
+ bool RevokeWithTag(Tag tag, unsigned w_idx) {
+ bool revoked = false;
+ std::unique_lock<OrtMutex> lock(mutex_);
+ Elem& e = array_[w_idx];
+ ElemState s = e.state.load(std::memory_order_relaxed);
+
+ // We have acquired a lock on the queue, synchronizing with
+ // operations aside from the PopFront fast-path. Synchronize with
+ // that by attempting the same kReady->kBusy transition via CAS.
+
+ if (s == ElemState::kReady &&
+ e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {
+ if (e.tag == tag) {
+ unsigned back = back_.load(std::memory_order_relaxed);
+ unsigned back_idx = back & kMask;
+ if (back_idx != w_idx) {
+ // Item is not at the back of the queue, mark it in-place as revoked
+ e.tag = Tag();
+ e.w = Work();
+ e.state.store(ElemState::kRevoked, std::memory_order_release);
+ revoked = true;
+ } else {
+ // Item being removed as still at the back; shift the back pointer over it,
+ // and bump the version number.
+ e.tag = Tag();
+ e.w = Work();
+ e.state.store(ElemState::kEmpty, std::memory_order_release);
+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);
+ revoked = true;
+ }
+ } else {
+ // Tag mismatch, i.e. work queue slot re-used
+ e.state.store(ElemState::kReady, std::memory_order_release);
+ }
+ }
+ return revoked;
+ }
+
+ // Size returns current queue size.
+ // Can be called by any thread at any time.
+ unsigned Size() const {
+ return SizeOrNotEmpty<true>();
+ }
+
+ // Empty tests whether container is empty.
+ // Can be called by any thread at any time.
+ bool Empty() const {
+ return SizeOrNotEmpty<false>() == 0;
+ }
+
+ // Delete all the elements from the queue.
+ void Flush() {
+ while (!Empty()) {
+ PopFront();
+ }
+ }
+
+ private:
+ static const unsigned kMask = kSize - 1;
+ static const unsigned kMask2 = (kSize << 1) - 1;
+
+ enum class ElemState : uint8_t {
+ kEmpty,
+ kBusy,
+ kReady,
+ kRevoked,
+ };
+
+ // Updates to an element are bracketed by a std::memory_order_acquire
+ // load from the state, and a std::memory_order_release store. Accesses
+ // to the front/back indices for the work queue use relaxed semantics,
+ // with the state of the elements being authoritative.
+ //
+ // TODO: Revisit whether there is a significant benefit for the current
+ // workloads in the complexity here.
+ struct Elem {
+ std::atomic<ElemState> state;
+ Tag tag;
+ Work w;
+ };
+
+ OrtMutex mutex_;
+
+ // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of
+ // front/back, respectively. The remaining bits contain modification counters
+ // that are incremented on Push operations. This allows us to (1) distinguish
+ // between empty and full conditions (if we would use log(kSize) bits for
+ // position, these conditions would be indistinguishable); (2) obtain
+ // consistent snapshot of front_/back_ for Size operation using the
+ // modification counters.
+ ORT_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> front_;
+ ORT_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> back_;
+ ORT_ALIGN_TO_AVOID_FALSE_SHARING Elem array_[kSize];
+
+ // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false,
+ // only whether the size is 0 is guaranteed to be correct.
+ // Can be called by any thread at any time.
+ template <bool NeedSizeEstimate>
+ unsigned SizeOrNotEmpty() const {
+ // Emptiness plays critical role in thread pool blocking. So we go to great
+ // effort to not produce false positives (claim non-empty queue as empty).
+ unsigned front = front_.load(std::memory_order_acquire);
+ for (;;) {
+ // Capture a consistent snapshot of front/tail.
+ unsigned back = back_.load(std::memory_order_acquire);
+ unsigned front1 = front_.load(std::memory_order_relaxed);
+ if (front != front1) {
+ front = front1;
+ std::atomic_thread_fence(std::memory_order_acquire);
+ continue;
+ }
+ if (NeedSizeEstimate) {
+ return CalculateSize(front, back);
+ }
+ // This value will be 0 if the queue is empty, and undefined otherwise.
+ unsigned maybe_zero = ((front ^ back) & kMask2);
+ // Queue size estimate must agree with maybe zero check on the queue
+ // empty/non-empty state.
+ eigen_assert((CalculateSize(front, back) == 0) == (maybe_zero == 0));
+ return maybe_zero;
+ }
+ }
+
+ EIGEN_ALWAYS_INLINE
+ unsigned CalculateSize(unsigned front, unsigned back) const {
+ int size = (front & kMask2) - (back & kMask2);
+ // Fix overflow.
+ if (size < 0)
+ size += 2 * kSize;
+ // Order of modification in push/pop is crafted to make the queue look
+ // larger than it is during concurrent modifications. E.g. push can
+ // increment size before the corresponding pop has decremented it.
+ // So the computed size can be up to kSize + 1, fix it.
+ if (size > static_cast<int>(kSize))
+ size = kSize;
+ return static_cast<unsigned>(size);
+ }
+
+ RunQueue(const RunQueue&) = delete;
+ void operator=(const RunQueue&) = delete;
+};
+
+static std::atomic<uint32_t> next_tag{1};
+
+template <typename Environment>
+class ThreadPoolTempl : public onnxruntime::concurrency::ExtendedThreadPoolInterface {
+
+ private:
+ struct PerThread;
+
+ static unsigned WorkerLoop(int id, Eigen::ThreadPoolInterface* param) {
+ // unsafe downcast
+ ThreadPoolTempl* this_ptr = (ThreadPoolTempl*)param;
+ this_ptr->WorkerLoop(id);
+ return 0;
+ }
+
+ public:
+ struct Tag {
+ constexpr Tag() : v_(0) {
+ }
+
+ Tag(uint32_t v) : v_(v) {
+ }
+
+ // Allocate a new tag to use to identify work items from a given
+ // thread in a parallel section. Ideally, threads will have
+ // unique tags, but re-use is not incorrect if the counter wraps
+ // (for intsance, if a long-running workload is calling into ORT
+ // from a fresh thread for each request). We must not re-use the
+ // default tag 0 which is used to identify work items added via
+ // Schedule as opposed to requests for help in parallel sections.
+
+ static Tag GetNext() {
+ Tag t = Tag(next_tag++);
+ if (t.v_ == 0) {
+ t = Tag(next_tag++);
+ }
+ return t;
+ }
+
+ uint32_t Get() const {
+ return v_;
+ }
+
+ bool operator==(Tag& other) const {
+ return v_ == other.v_;
+ }
+
+ uint32_t v_ = 0;
+ };
+
+ typedef std::function<void()> Task;
+ typedef RunQueue<Task, Tag, 1024> Queue;
+#ifdef _WIN32
+ using CHAR_TYPE = wchar_t;
+#else
+ using CHAR_TYPE = char;
+#endif
+ ThreadPoolTempl(const CHAR_TYPE* name, int num_threads, bool allow_spinning, Environment& env,
+ const ThreadOptions& thread_options)
+ : env_(env),
+ num_threads_(num_threads),
+ allow_spinning_(allow_spinning),
+ set_denormal_as_zero_(thread_options.set_denormal_as_zero),
+ worker_data_(num_threads),
+ all_coprimes_(num_threads),
+ blocked_(0),
+ done_(false),
+ cancelled_(false) {
+
+ // Calculate coprimes of all numbers [1, num_threads].
+ // Coprimes are used for random walks over all threads in Steal
+ // and NonEmptyQueueIndex. Iteration is based on the fact that if we take
+ // a random starting thread index t and calculate num_threads - 1 subsequent
+ // indices as (t + coprime) % num_threads, we will cover all threads without
+ // repetitions (effectively getting a presudo-random permutation of thread
+ // indices).
+ for (int i = 1; i <= num_threads_; ++i) {
+ all_coprimes_.emplace_back(i);
+ ComputeCoprimes(i, &all_coprimes_.back());
+ }
+
+ // Allocate space for per-thread bits to indicate which threads to consider
+ // preferable for pushing work. We use a regular array given that a std::vector
+ // cannot contain std::atomic.
+ num_hint_words_ = static_cast<int>((num_threads_ + bits_per_hint_word_ - 1) / bits_per_hint_word_);
+ good_worker_hints_ = onnxruntime::make_unique<std::atomic<uint64_t>[]>(num_hint_words_);
+
+ worker_data_.resize(num_threads_);
+ for (int i = 0; i < num_threads_; i++) {
+ worker_data_[i].thread.reset(env_.CreateThread(name, i, WorkerLoop, this, thread_options));
+ }
+ }
+
+ ~ThreadPoolTempl() override {
+ done_ = true;
+
+ // Now if all threads block without work, they will start exiting.
+ // But note that threads can continue to work arbitrary long,
+ // block, submit new work, unblock and otherwise live full life.
+ if (!cancelled_) {
+ WakeAllWorkersForExit();
+ } else {
+ // Since we were cancelled, there might be entries in the queues.
+ // Empty them to prevent their destructor from asserting.
+ for (size_t i = 0; i < worker_data_.size(); i++) {
+ worker_data_[i].queue.Flush();
+ }
+ }
+ // Join threads explicitly (by destroying) to avoid destruction order within
+ // this class.
+ for (size_t i = 0; i < worker_data_.size(); ++i) worker_data_[i].thread.reset();
+ }
+
+ // Run fn(). Ordinarily, the function will be added to the thread pool and executed
+ // by a worker thread. If the thread pool rejects the work then fn() will instead
+ // execute synchronously during Schedule(fn). Currently the thread pool will only
+ // reject work if the queue of pending work is full.
+
+ void Schedule(std::function<void()> fn) override {
+ PerThread* pt = GetPerThread();
+ if (pt->pool == this) {
+ // Worker thread of this pool, push onto the thread's queue.
+ Queue& q = worker_data_[pt->thread_id].queue;
+ fn = q.PushFront(std::move(fn));
+ } else {
+ // A free-standing thread (or worker of another pool), push onto a random
+ // queue.
+ int q_idx = Rand(&pt->rand) % num_threads_;
+ WorkerData &td = worker_data_[q_idx];
+ Queue& q = td.queue;
+ fn = q.PushBack(std::move(fn));
+ if (!fn) {
+ // The queue accepted the work; ensure that the thread will pick it up
+ td.EnsureAwake();
+ }
+ }
+
+ // Run the work directly if the queue rejected the work
+ if (fn) fn();
+ }
+
+// The thread pool maintains a set of hints for which threads will be good to distribute
+// work to. A thread is considered "good" if it is actively spinning, meaning both that
+// it is not busy with existing work, and that it should respond quickly to the addition
+// of new work.
+
+void SetGoodWorkerHint(int idx, bool is_good) {
+ assert(idx >= 0 && idx < num_threads_);
+ std::atomic<uint64_t>& u64 = good_worker_hints_[idx / bits_per_hint_word_];
+ uint64_t bit = 1ull << (idx % bits_per_hint_word_);
+ uint64_t saw, want;
+ do {
+ saw = u64.load();
+ want = is_good ? (saw|bit) : (saw&~bit);
+ } while (!u64.compare_exchange_weak(saw, want));
+}
+
+// Retrieve hints for up to n threads to distribute work to. Threads in good_hints
+// pass a best-effort check to identify spinning threads via the good_worker_hints_
+// bitmap. Threads in alt_hint do not pass that test, but are distinct from those in
+// good_hints, letting the caller avoid distributing more than one work item to
+// any individual thread.
+
+void GetGoodWorkerHints(unsigned n, std::vector<unsigned>& good_hints, std::vector<unsigned>& alt_hints) {
+ PerThread* pt = GetPerThread();
+ unsigned need_alt = n;
+ good_hints.clear();
+ alt_hints.clear();
+
+ // Iterate through the words of hints, starting from a pseudo-randomly chosen
+ // base. This aims to distribute work across large machines in cases we
+ // have multiple threads scheduling work concurrently.
+
+ unsigned base = Rand(&pt->rand) % num_hint_words_;
+ for (unsigned i = 0u; n && (i < num_hint_words_); i++) {
+ int u64_idx = (base + i) % num_hint_words_;
+ std::atomic<uint64_t>* u64 = &good_worker_hints_[u64_idx];
+ uint64_t saw = u64->load();
+ uint64_t want = saw;
+
+ // Pick up to n bits that are set in the current word
+ for (unsigned j = 0u; n && (j < bits_per_hint_word_); j++) {
+ uint64_t bit = 1ull << j;
+ int thread = u64_idx * bits_per_hint_word_ + j;
+ if (saw & bit) {
+ good_hints.push_back(thread);
+ want &= ~bit;
+ n--;
+ } else if (need_alt && thread < num_threads_) {
+ alt_hints.push_back(thread);
+ need_alt--;
+ }
+ }
+
+ // Best-effort attempt to remove the hints. We should measure the impact of
+ // contention here, but the intuition is that if we conflict on the CAS then the
+ // machine is likely to be busy in any case, and we will have queuing on the
+ // work items.
+ u64->compare_exchange_strong(saw, want);
+ }
+}
+
+//......................................................................
+//
+// Parallel sections
+// -----------------
+//
+// Allocate a new ThreadPoolParallelSection, owned by the returned
+// unique_ptr. The explicit deleter avoids the Eigen-specific
+// definition of ThreadPoolParallelSection needing to be avilable in
+// threadpool.h where the user-facing parallel section API is defined.
+
+std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)> AllocateParallelSection() override {
+ return std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)>
+ (new ThreadPoolParallelSection,
+ [](ThreadPoolParallelSection *tps) {
+ delete tps;
+ });
+}
+
+// Start a parallel section, using a caller-provided
+// ThreadPoolParallelSection for maintaining the per-section state.
+// Starting a parallel section is just book-keeping; threads are
+// "summoned" to help with the parallel section once it enters
+// parallel loops. The threads are then retained until the end of the
+// section, being re-used over subsequent loops.
+
+void StartParallelSectionInternal(PerThread &pt,
+ ThreadPoolParallelSection &ps) {
+ assert((!pt.leading_par_section) && "Nested parallelism not supported");
+ assert((!ps.active) && "Starting parallel section, but active already");
+ pt.leading_par_section = true;
+ if (!pt.tag.Get()) {
+ pt.tag = Tag::GetNext();
+ }
+ ps.active = true;
+}
+
+void StartParallelSection(ThreadPoolParallelSection &ps) override {
+ PerThread* pt = GetPerThread();
+ StartParallelSectionInternal(*pt, ps);
+}
+
+// End a parallel section, waiting for all worker threads to exit from
+// section. Hence, on return, the ThreadPoolParallelSection object
+// can be dealloacted.
+
+void EndParallelSectionInternal(PerThread &pt,
+ ThreadPoolParallelSection &ps) {
+ assert((pt.leading_par_section) && "Ending parallel section, but none started");
+ assert((ps.active) && "Ending parallel section, but not active");
+ pt.leading_par_section = false;
+
+ // Notify workers to exit from the section
+ ps.active = false;
+
+ // Attempt to revoke any tasks that were sent to workers but not
+ // started.
+ unsigned tasks_started = static_cast<unsigned>(ps.tasks.size());
+ unsigned tasks_revoked = 0;
+ while (!ps.tasks.empty()) {
+ const auto& item = ps.tasks.back();
+ Queue& q = worker_data_[item.first].queue;
+ if (q.RevokeWithTag(pt.tag, item.second)) {
+ tasks_revoked++;
+ }
+ ps.tasks.pop_back();
+ }
+
+ // Wait for workers to exit ParLoopWorker
+ auto tasks_to_wait_for = tasks_started - tasks_revoked;
+ while (ps.tasks_finished < tasks_to_wait_for) {
+ onnxruntime::concurrency::SpinPause();
+ }
+
+ // Clear status to allow the ThreadPoolParallelSection to be
+ // re-used.
+ ps.tasks_finished = 0;
+}
+
+void EndParallelSection(ThreadPoolParallelSection &ps) override {
+ PerThread* pt = GetPerThread();
+ EndParallelSectionInternal(*pt, ps);
+}
+
+//......................................................................
+//
+// Parallel loops
+// --------------
+//
+// Ensure that the ThreadPoolParallelSection has sufficient workers to
+// execute a loop with degree of parallelism n. We track the number
+// of workers already avaiable to the parallel section, prior to
+// submitting tasks to the work queues to make up the total.
+//
+// Each worker will call in to worker_fn(idx) with a per-worker thread
+// ID. Note there are different levels of indirection here:
+//
+// - In a single-loop parallel section, worker_fn will directly
+// execute the threadpool.cc code that implements the parallel loop.
+//
+// - In a multi-loop parallel section, worker_fn is an intermediate
+// function that is long-lived (i.e., that lasts until the end of
+// the parallel section, as opposed to just a single loop's
+// duration).
+
+void SummonWorkers(PerThread &pt,
+ ThreadPoolParallelSection &ps,
+ unsigned n,
+ const std::function<void(unsigned)> &worker_fn) {
+ // Wrap the user's worker function with one that allocates a unique
+ // worker index for the loop, and synchronizes (as the last step)
+ // with the exit path in EndParallelSection. In principle we could
+ // allocate worker IDs during the loop below and capture them by
+ // value. However, the costs of creating distinct lambda for each
+ // iteration appeared more costly than the cost of synchronization
+ // on a shared counter.
+ auto call_worker_fn = [&ps, worker_fn]() {
+ unsigned my_idx = ++ps.worker_idx;
+ worker_fn(my_idx);
+ // After the assignment to ps.tasks_finished, the stack-allocated
+ // ThreadPoolParallelSection object may be destroyed.
+ ps.tasks_finished++;
+ };
+
+ // Identify whether we need to create additional workers.
+ // Throughout the threadpool implementation, degrees of parallelism
+ // ("n" here) refer to the total parallelism including the main
+ // thread. Hence we consider the number of existing tasks + 1.
+ unsigned current_dop = static_cast<unsigned>(ps.tasks.size()) + 1;
+ if (n > current_dop) {
+ unsigned extra_needed = n - current_dop;
+
+ // Obtain hints for which worker threads to push the tasks to.
+ // This uses a best-effort assessment of which threads are
+ // spinning.
+ std::vector<unsigned> good_hints, alt_hints;
+ GetGoodWorkerHints(extra_needed, good_hints, alt_hints);
+
+ // Create the additional tasks, and push them to workers.
+ for (auto i = 0u; i < extra_needed; i++) {
+ Task t;
+ int q_idx;
+ if (i < good_hints.size()) {
+ q_idx = good_hints[i];
+ } else {
+ auto alt_i = i - static_cast<unsigned>(good_hints.size());
+ if (alt_i < alt_hints.size()) {
+ q_idx = alt_hints[alt_i];
+ } else {
+ q_idx = Rand(&pt.rand) % num_threads_;
+ }
+ }
+
+ // If the worker's queue accepts the task, then record it in
+ // the vector of tasks that we will need to synchronize with on
+ // exiting the parallel section. If the queue rejects the task
+ // (perhaps because it is full) then we take no further action:
+ // in a parallel loop we will always be running work on the
+ // main thread, providing progress.
+ WorkerData& td = worker_data_[q_idx];
+ Queue& q = td.queue;
+ unsigned w_idx;
+ t = q.PushBackWithTag(call_worker_fn, pt.tag, w_idx);
+ if (!t) {
+ ps.tasks.push_back({q_idx, w_idx});
+ td.EnsureAwake();
+ }
+ }
+ }
+}
+
+// Run a single parallel loop in an existing parallel section. This
+// maps directly onto SummonWorkers to create sufficient worker
+// threads for the desired degree of parallelism, followed by
+// dispatching the loop to those workers.
+
+void RunInParallelSection(ThreadPoolParallelSection &ps,
+ std::function<void(unsigned idx)> fn,
+ unsigned n) override {
+ PerThread* pt = GetPerThread();
+ assert(pt->leading_par_section && "RunInParallel, but not in parallel section");
+ assert((n > 1) && "Trivial parallel section; should be avoided by caller");
+
+ // Publish the work to any existing workers in the parallel
+ // section, and ensure it is visible to any new threads created
+ // below.
+ assert((!ps.current_loop) && "RunInParallelSection, but loop already active");
+ ThreadPoolLoop loop{std::move(fn), n};
+ ps.current_loop = &loop;
+
+ // Increase the worker count if needed. Each worker will pick up
+ // loops to execute from the current parallel section.
+ const auto worker_fn = [&ps](unsigned my_idx) {
+ while (ps.active) {
+ if (!ps.current_loop) {
+ onnxruntime::concurrency::SpinPause();
+ } else {
+ ps.workers_in_loop++;
+ ThreadPoolLoop *work_item = ps.current_loop;
+ if (work_item && my_idx < work_item->threads_needed) {
+ work_item->fn(my_idx);
+ }
+ ps.workers_in_loop--;
+ }
+ }
+ };
+ SummonWorkers(*pt, ps, n, worker_fn);
+
+ // Run work in the main thread
+ loop.fn(0);
+
+ // Wait for workers to exit the loop
+ ps.current_loop = 0;
+ while (ps.workers_in_loop) {
+ onnxruntime::concurrency::SpinPause();
+ }
+}
+
+// Run a single parallel loop _without_ a parallel section. This is a
+// special case of RunInParallelSection, avoiding code paths for
+// handing off multiple loops to the pool of workers.
+
+void RunInParallel(std::function<void(unsigned idx)> fn, unsigned n) override {
+ PerThread *pt = GetPerThread();
+ ThreadPoolParallelSection ps;
+ StartParallelSectionInternal(*pt, ps);
+
+ // Summon workers to run the function (n is the desired maximum
+ // degree of parallelism, including the main thread). Unlike the
+ // multi-loop RunInParallelSection, this single-loop worker can run
+ // fn directly without needing to receive it via ps.current_loop.
+ SummonWorkers(*pt, ps, n, fn);
+
+ // Run work in the main thread
+ fn(0);
+
+ // Wait for workers to exit the parallel section and hence to have
+ // completed the loop (i.e., ps.tasks_finished matches the number of
+ // tasks that have been created less the number successfully
+ // revoked).
+ EndParallelSectionInternal(*pt, ps);
+}
+
+void Cancel() override {
+ cancelled_ = true;
+ // If done_ is true, which means this object is being destructing.
+ // Therefore worker_data_[i].thread could be NULL.
+ if (!done_) {
+ done_ = true;
+ // Let each thread know it's been cancelled.
+ for (size_t i = 0; i < worker_data_.size(); i++) {
+ assert(worker_data_[i].thread != nullptr);
+ worker_data_[i].thread->OnCancel();
+ }
+ }
+
+ // Wake up the threads without work to let them exit on their own.
+ WakeAllWorkersForExit();
+}
+
+int NumThreads() const EIGEN_FINAL {
+ return num_threads_;
+}
+
+int CurrentThreadId() const EIGEN_FINAL {
+ const PerThread* pt = const_cast<ThreadPoolTempl*>(this)->GetPerThread();
+ if (pt->pool == this) {
+ return pt->thread_id;
+ }
+ return -1;
+}
+
+ private:
+
+#ifdef NDEBUG
+ void AssertBounds(int, int) {
+ }
+#else
+ void AssertBounds(int start, int end) {
+ assert(start >= 0);
+ assert(start < end); // non-zero sized partition
+ assert(end <= num_threads_);
+ }
+#endif
+
+ void ComputeCoprimes(int N, Eigen::MaxSizeVector<unsigned>* coprimes) {
+ for (int i = 1; i <= N; i++) {
+ unsigned a = i;
+ unsigned b = N;
+ // If GCD(a, b) == 1, then a and b are coprimes.
+ while (b != 0) {
+ unsigned tmp = a;
+ a = b;
+ b = tmp % b;
+ }
+ if (a == 1) {
+ coprimes->push_back(i);
+ }
+ }
+ }
+
+ typedef typename Environment::EnvThread Thread;
+ struct WorkerData;
+
+ // PerThread objects are allocated in thread-local storage and allocated
+ // on the thread's first call to GetPerThread. The object should
+ // remain trivially-destructable, with other state placed in the
+ // WorkerData objects that are allocated and cleaned-up explicitly.
+ //
+ // PerThread objects are allocated for all threads that submit work to
+ // the thread pool, in addition to threads within the pool.
+ //
+ // In contrast, the WorkerData objects are allocated only for the
+ // threads in the pool, and their lifetime is managed along with the
+ // pool.
+
+ struct PerThread {
+ constexpr PerThread() : pool(nullptr) {
+ }
+ ThreadPoolTempl* pool; // Parent pool, or null for normal threads.
+ uint64_t rand{0}; // Random generator state.
+ int thread_id{-1}; // Worker thread index in pool.
+ Tag tag{}; // Work item tag used to identify this thread.
+ bool leading_par_section{false}; // Leading a parallel section (used only for asserts)
+ };
+
+ static_assert(std::is_trivially_destructible<PerThread>::value,
+ "Per-thread state should be trivially destructible");
+
+ struct WorkerData {
+ constexpr WorkerData() : thread(), queue() {
+ }
+ std::unique_ptr<Thread> thread;
+ Queue queue;
+
+ // Each thread has a status, available read-only without locking, and protected
+ // by the mutex field below for updates. The status is used for three
+ // purposes:
+ //
+ // 1. To identify threads that are good candidates to push work to.
+ // We prefer to push work to threads that are actively spinning (no need
+ // for an OS wake-up, and no need for current work to finish). After that, we
+ // prefer to push work to threads that are blocked (no need to wait for the
+ // current work to finish).
+ //
+ // 2. To identify threads that are good candidates to steal work from. We
+ // prefer to steal work from threads that are active outside the worker loop.
+ // This avoids "snatching" new work away from a thread that has just been
+ // given it but not yet noticed.
+ //
+ // 3. When pushing work to a thread, we use the status read-only to identify
+ // when we need to wake the thread. This read-only check avoids the
+ // need for mutex / condvar operations in the case where the thread pool
+ // remains busy.
+
+ enum class ThreadStatus : uint8_t {
+ Spinning, // Spinning in the work loop, and other cases (initialization) where
+ // the thread will soon be in the loop
+ Active, // Running user code, not waiting for work
+ Blocking, // In the process of blocking; may no longer notice work pushed to it
+ Blocked, // Blocked on cv
+ Waking, // Not yet back in the worker loop, but wake-up notification sent
+ };
+
+ ThreadStatus GetStatus() const {
+ return status;
+ }
+
+ // State transitions, called from other threads
+
+ void EnsureAwake() {
+ ThreadStatus seen = status;
+ if (seen == ThreadStatus::Blocking ||
+ seen == ThreadStatus::Blocked) {
+ std::unique_lock<OrtMutex> lk(mutex);
+ // Blocking state exists only transiently during the SetBlock() method
+ // while holding the lock. We may observe it at the start of this
+ // function, but after acquiring the lock then the target thread
+ // will either be blocked or not.
+ seen = status;
+ assert(seen != ThreadStatus::Blocking);
+ if (seen == ThreadStatus::Blocked) {
+ status = ThreadStatus::Waking;
+ cv.notify_one();
+ }
+ }
+ }
+
+ // State transitions, called only from the thread itself
+
+ void SetActive() {
+ std::unique_lock<OrtMutex> lk(mutex);
+ status = ThreadStatus::Active;
+ }
+
+ void SetSpinning() {
+ std::unique_lock<OrtMutex> lk(mutex);
+ status = ThreadStatus::Spinning;
+ }
+
+ void SetBlocked(std::function<bool()> should_block,
+ std::function<void()> post_block) {
+ std::unique_lock<OrtMutex> lk(mutex);
+ assert(status == ThreadStatus::Spinning);
+ status = ThreadStatus::Blocking;
+ if (should_block()) {
+ status = ThreadStatus::Blocked;
+ while (status == ThreadStatus::Blocked) {
+ cv.wait(lk);
+ }
+ post_block();
+ }
+ status = ThreadStatus::Spinning;
+ }
+
+ private:
+ std::atomic<ThreadStatus> status{ThreadStatus::Spinning};
+ OrtMutex mutex;
+ OrtCondVar cv;
+ };
+
+ Environment& env_;
+ const int num_threads_;
+ const bool allow_spinning_;
+ const bool set_denormal_as_zero_;
+ Eigen::MaxSizeVector<WorkerData> worker_data_;
+ Eigen::MaxSizeVector<Eigen::MaxSizeVector<unsigned>> all_coprimes_;
+ std::atomic<unsigned> blocked_; // Count of blocked workers, used as a termination condition
+ std::atomic<bool> done_;
+ std::atomic<bool> cancelled_;
+
+ // Allow control over how many bits to use in each entry in good_worker_hints_.
+ // We reduce this below the full 64-bit word size for two reasons. First, it
+ // helps test coverage on machines without 64 vCPUS. Second, it lets us
+ // reduce contention by having different threads start work searching for hints
+ // at different locations in the bitmap.
+
+ static const unsigned bits_per_hint_word_ = 4;
+ unsigned num_hint_words_;
+ std::unique_ptr<std::atomic<uint64_t>[]> good_worker_hints_;
+
+ // Wake any blocked workers so that they can cleanly exit WorkerLoop(). For an
+ // abrupt exit, cancelled_==true and threads will exit their worker loops. For
+ // a clean exit, each thread will observe (1) done_ set, indicating that the
+ // destructor has been called, (2) all threads blocked, and (3) no
+ // items in the work queues.
+
+ void WakeAllWorkersForExit() {
+ for (auto &td: worker_data_) {
+ td.EnsureAwake();
+ }
+ }
+
+ // Main worker thread loop.
+ void WorkerLoop(int thread_id) {
+ PerThread* pt = GetPerThread();
+ WorkerData& td = worker_data_[thread_id];
+ Queue& q = td.queue;
+ bool should_exit = false;
+ pt->pool = this;
+ pt->rand = GlobalThreadIdHash();
+ pt->thread_id = thread_id;
+
+ assert(td.GetStatus() == WorkerData::ThreadStatus::Spinning);
+ SetGoodWorkerHint(thread_id, true /* Is good */);
+
+ const int log2_spin = 20;
+ const int spin_count = allow_spinning_ ? (1ull<<log2_spin) : 0;
+ const int steal_count = spin_count/100;
+
+ SetDenormalAsZero(set_denormal_as_zero_);
+
+ while (!cancelled_ && !should_exit) {
+ Task t = q.PopFront();
+ if (!t) {
+ // Spin waiting for work. We indicate, via SetGOodWorkerHint that we are
+ // spinning. This will bias other threads toward pushing work to our queue.
+ // In addition, priodically make a best-effort attempt to steal from other
+ // threads which are not themselves spinning.
+
+ SetGoodWorkerHint(thread_id, true);
+ for (int i = 0; i < spin_count && !t && !cancelled_ && !done_; i++) {
+ t = ((i+1)%steal_count == 0) ? TrySteal() : q.PopFront();
+ onnxruntime::concurrency::SpinPause();
+ }
+ SetGoodWorkerHint(thread_id, false);
+
+ if (!t) {
+ // No work passed to us while spinning; make a further full attempt to
+ // steal work from other threads prior to blocking.
+ if (num_threads_ != 1) {
+ t = Steal(true /* true => check all queues */);
+ }
+ if (!t) {
+ td.SetBlocked(
+ // Pre-block test
+ [&]() -> bool {
+ bool should_block = true;
+ // We already did a best-effort emptiness check when stealing; now
+ // do a full check prior to blocking.
+ int victim = NonEmptyQueueIndex();
+ if (victim != -1) {
+ should_block = false;
+ if (!cancelled_) {
+ t = worker_data_[victim].queue.PopBack();
+ }
+ }
+ // Number of blocked threads is used as termination condition.
+ // If we are shutting down and all worker threads blocked without work,
+ // that's we are done.
+ if (should_block) {
+ blocked_++;
+ if (done_ && blocked_ == static_cast<unsigned>(num_threads_)) {
+ should_block = false;
+ // Almost done, but need to re-check queues.
+ // Consider that all queues are empty and all worker threads are preempted
+ // right after incrementing blocked_ above. Now a free-standing thread
+ // submits work and calls destructor (which sets done_). If we don't
+ // re-check queues, we will exit leaving the work unexecuted.
+ if (NonEmptyQueueIndex() != -1) {
+ // Note: we must not pop from queues before we decrement blocked_,
+ // otherwise the following scenario is possible. Consider that instead
+ // of checking for emptiness we popped the only element from queues.
+ // Now other worker threads can start exiting, which is bad if the
+ // work item submits other work. So we just check emptiness here,
+ // which ensures that all worker threads exit at the same time.
+ blocked_--;
+ } else {
+ should_exit = true;
+ }
+ }
+ }
+ return should_block;
+ },
+ // Post-block update (executed only if we blocked)
+ [&]() {
+ blocked_--;
+ });
+ }
+ }
+ }
+ if (t) {
+ td.SetActive();
+ t();
+ td.SetSpinning();
+ }
+ }
+
+ // Whichever thread(s) observe the termination conditions are responsible for waking
+ // any other threads that have remained blocked.
+ if (should_exit) {
+ WakeAllWorkersForExit();
+ }
+ }
+
+ // Steal tries to steal work from other worker threads in the range [start,
+ // limit) in best-effort manner. We make two passes over the threads:
+ //
+ // - round 0 : we attempt to steal from threads that are running in
+ // user code (ThreadStatus::Active). The intuition behind this is that
+ // the thread is busy with other work, and that by preferring to
+ // steel from busy victims we will avoid "snatching" work from a
+ // thread which is just about to notice the work itself.
+ //
+ // - round 1 : we steal work from any thread, including those which claim
+ // to be spinning. In these cases, even though the victim thread is
+ // looking for work itself, it may have been pre-empted.
+
+ Task Steal(bool check_all) {
+ PerThread* pt = GetPerThread();
+ unsigned size = static_cast<unsigned>(num_threads_);
+ unsigned r = Rand(&pt->rand);
+ unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()];
+
+ for (int round = 0; round < 2; round++) {
+ unsigned victim = r % size;
+ for (unsigned i = 0; i < size; i++) {
+ assert(victim < size);
+ if (round == 1 ||
+ worker_data_[victim].GetStatus() == WorkerData::ThreadStatus::Active) {
+ Task t = worker_data_[victim].queue.PopBack();
+ if (t) {
+ return t;
+ }
+ }
+ if (!check_all) {
+ return Task();
+ }
+ victim += inc;
+ if (victim >= size) {
+ victim -= size;
+ }
+ }
+ }
+
+ return Task();
+ }
+
+ Task TrySteal() {
+ return Steal(false);
+ }
+
+ int NonEmptyQueueIndex() {
+ PerThread* pt = GetPerThread();
+ const unsigned size = static_cast<unsigned>(worker_data_.size());
+ unsigned r = Rand(&pt->rand);
+ unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()];
+ unsigned victim = r % size;
+ for (unsigned i = 0; i < size; i++) {
+ if (!worker_data_[victim].queue.Empty()) {
+ return victim;
+ }
+ victim += inc;
+ if (victim >= size) {
+ victim -= size;
+ }
+ }
+ return -1;
+ }
+
+ static EIGEN_STRONG_INLINE uint64_t GlobalThreadIdHash() {
+ return std::hash<std::thread::id>()(std::this_thread::get_id());
+ }
+
+ static EIGEN_STRONG_INLINE PerThread* GetPerThread() {
+ static thread_local PerThread per_thread_;
+ PerThread* pt = &per_thread_;
+ return pt;
+ }
+
+ static EIGEN_STRONG_INLINE unsigned Rand(uint64_t* state) {
+ uint64_t current = *state;
+ // Update the internal state
+ *state = current * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL;
+ // Generate the random output (using the PCG-XSH-RS scheme)
+ return static_cast<unsigned>((current ^ (current >> 22)) >> (22 + (current >> 61)));
+ }
+};
+
+ } // namespace concurrency
+
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/platform/ort_mutex.h b/include/onnxruntime/core/platform/ort_mutex.h
new file mode 100644
index 0000000..e24665f
--- /dev/null
+++ b/include/onnxruntime/core/platform/ort_mutex.h
@@ -0,0 +1,189 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#ifdef _WIN32
+#include <Windows.h>
+#include <mutex>
+namespace onnxruntime {
+// Q: Why OrtMutex is better than std::mutex
+// A: OrtMutex supports static initialization but std::mutex doesn't. Static initialization helps us prevent the "static
+// initialization order problem".
+
+// Q: Why std::mutex can't make it?
+// A: VC runtime has to support Windows XP at ABI level. But we don't have such requirement.
+
+// Q: Is OrtMutex faster than std::mutex?
+// A: Sure
+
+class OrtMutex {
+ private:
+ SRWLOCK data_ = SRWLOCK_INIT;
+
+ public:
+ constexpr OrtMutex() = default;
+ // SRW locks do not need to be explicitly destroyed.
+ ~OrtMutex() = default;
+ OrtMutex(const OrtMutex&) = delete;
+ OrtMutex& operator=(const OrtMutex&) = delete;
+ void lock() { AcquireSRWLockExclusive(native_handle()); }
+ bool try_lock() noexcept { return TryAcquireSRWLockExclusive(native_handle()) == TRUE; }
+ void unlock() noexcept { ReleaseSRWLockExclusive(native_handle()); }
+ using native_handle_type = SRWLOCK*;
+
+ __forceinline native_handle_type native_handle() { return &data_; }
+};
+
+class OrtCondVar {
+ CONDITION_VARIABLE native_cv_object = CONDITION_VARIABLE_INIT;
+
+ public:
+ constexpr OrtCondVar() noexcept = default;
+ ~OrtCondVar() = default;
+
+ OrtCondVar(const OrtCondVar&) = delete;
+ OrtCondVar& operator=(const OrtCondVar&) = delete;
+
+ void notify_one() noexcept { WakeConditionVariable(&native_cv_object); }
+ void notify_all() noexcept { WakeAllConditionVariable(&native_cv_object); }
+
+ void wait(std::unique_lock<OrtMutex>& lk) {
+ if (SleepConditionVariableSRW(&native_cv_object, lk.mutex()->native_handle(), INFINITE, 0) != TRUE) {
+ std::terminate();
+ }
+ }
+ template <class _Predicate>
+ void wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred);
+
+ /**
+ * returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns
+ * cv_status::no_timeout.
+ * @param cond_mutex A unique_lock<OrtMutex> object.
+ * @param rel_time A chrono::duration object that specifies the amount of time before the thread wakes up.
+ * @return returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns
+ * cv_status::no_timeout
+ */
+ template <class Rep, class Period>
+ std::cv_status wait_for(std::unique_lock<OrtMutex>& cond_mutex, const std::chrono::duration<Rep, Period>& rel_time);
+ using native_handle_type = CONDITION_VARIABLE*;
+
+ native_handle_type native_handle() { return &native_cv_object; }
+
+ private:
+ void timed_wait_impl(std::unique_lock<OrtMutex>& __lk,
+ std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>);
+};
+
+template <class _Predicate>
+void OrtCondVar::wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred) {
+ while (!__pred()) wait(__lk);
+}
+
+template <class Rep, class Period>
+std::cv_status OrtCondVar::wait_for(std::unique_lock<OrtMutex>& cond_mutex,
+ const std::chrono::duration<Rep, Period>& rel_time) {
+ // TODO: is it possible to use nsync_from_time_point_ ?
+ using namespace std::chrono;
+ if (rel_time <= duration<Rep, Period>::zero())
+ return std::cv_status::timeout;
+ using SystemTimePointFloat = time_point<system_clock, duration<long double, std::nano> >;
+ using SystemTimePoint = time_point<system_clock, nanoseconds>;
+ SystemTimePointFloat max_time = SystemTimePoint::max();
+ steady_clock::time_point steady_now = steady_clock::now();
+ system_clock::time_point system_now = system_clock::now();
+ if (max_time - rel_time > system_now) {
+ nanoseconds remain = duration_cast<nanoseconds>(rel_time);
+ if (remain < rel_time)
+ ++remain;
+ timed_wait_impl(cond_mutex, system_now + remain);
+ } else
+ timed_wait_impl(cond_mutex, SystemTimePoint::max());
+ return steady_clock::now() - steady_now < rel_time ? std::cv_status::no_timeout : std::cv_status::timeout;
+}
+} // namespace onnxruntime
+#else
+#include "nsync.h"
+#include <mutex> //for unique_lock
+#include <condition_variable> //for cv_status
+namespace onnxruntime {
+
+class OrtMutex {
+ nsync::nsync_mu data_ = NSYNC_MU_INIT;
+
+ public:
+ constexpr OrtMutex() = default;
+ ~OrtMutex() = default;
+ OrtMutex(const OrtMutex&) = delete;
+ OrtMutex& operator=(const OrtMutex&) = delete;
+
+ void lock() { nsync::nsync_mu_lock(&data_); }
+ bool try_lock() noexcept { return nsync::nsync_mu_trylock(&data_) == 0; }
+ void unlock() noexcept { nsync::nsync_mu_unlock(&data_); }
+
+ using native_handle_type = nsync::nsync_mu*;
+ native_handle_type native_handle() { return &data_; }
+};
+
+class OrtCondVar {
+ nsync::nsync_cv native_cv_object = NSYNC_CV_INIT;
+
+ public:
+ constexpr OrtCondVar() noexcept = default;
+
+ ~OrtCondVar() = default;
+ OrtCondVar(const OrtCondVar&) = delete;
+ OrtCondVar& operator=(const OrtCondVar&) = delete;
+
+ void notify_one() noexcept { nsync::nsync_cv_signal(&native_cv_object); }
+ void notify_all() noexcept { nsync::nsync_cv_broadcast(&native_cv_object); }
+
+ void wait(std::unique_lock<OrtMutex>& lk);
+ template <class _Predicate>
+ void wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred);
+
+ /**
+ * returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns
+ * cv_status::no_timeout.
+ * @param cond_mutex A unique_lock<OrtMutex> object.
+ * @param rel_time A chrono::duration object that specifies the amount of time before the thread wakes up.
+ * @return returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns
+ * cv_status::no_timeout
+ */
+ template <class Rep, class Period>
+ std::cv_status wait_for(std::unique_lock<OrtMutex>& cond_mutex, const std::chrono::duration<Rep, Period>& rel_time);
+ using native_handle_type = nsync::nsync_cv*;
+ native_handle_type native_handle() { return &native_cv_object; }
+
+ private:
+ void timed_wait_impl(std::unique_lock<OrtMutex>& __lk,
+ std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>);
+};
+
+template <class _Predicate>
+void OrtCondVar::wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred) {
+ while (!__pred()) wait(__lk);
+}
+
+template <class Rep, class Period>
+std::cv_status OrtCondVar::wait_for(std::unique_lock<OrtMutex>& cond_mutex,
+ const std::chrono::duration<Rep, Period>& rel_time) {
+ // TODO: is it possible to use nsync_from_time_point_ ?
+ using namespace std::chrono;
+ if (rel_time <= duration<Rep, Period>::zero())
+ return std::cv_status::timeout;
+ using SystemTimePointFloat = time_point<system_clock, duration<long double, std::nano> >;
+ using SystemTimePoint = time_point<system_clock, nanoseconds>;
+ SystemTimePointFloat max_time = SystemTimePoint::max();
+ steady_clock::time_point steady_now = steady_clock::now();
+ system_clock::time_point system_now = system_clock::now();
+ if (max_time - rel_time > system_now) {
+ nanoseconds remain = duration_cast<nanoseconds>(rel_time);
+ if (remain < rel_time)
+ ++remain;
+ timed_wait_impl(cond_mutex, system_now + remain);
+ } else
+ timed_wait_impl(cond_mutex, SystemTimePoint::max());
+ return steady_clock::now() - steady_now < rel_time ? std::cv_status::no_timeout : std::cv_status::timeout;
+}
+}; // namespace onnxruntime
+#endif
diff --git a/include/onnxruntime/core/platform/threadpool.h b/include/onnxruntime/core/platform/threadpool.h
new file mode 100644
index 0000000..126dd13
--- /dev/null
+++ b/include/onnxruntime/core/platform/threadpool.h
@@ -0,0 +1,441 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+/* Modifications Copyright (c) Microsoft. */
+
+#pragma once
+#include <string>
+#include <vector>
+#include <functional>
+#include <memory>
+#include "core/common/common.h"
+#include "core/platform/env.h"
+
+#include <functional>
+#include <memory>
+
+// ORT thread pool overview
+// ------------------------
+//
+// The ORT thread pool implementation is split into two layers. This
+// file provides the high-level component. See the accompanying
+// comments in EigenNonBlockingThreadPool.h for the low-level
+// component.
+//
+// threadpool.h defines the user-facing functions for use in
+// operators. The main abstraction are parallel loops
+// (ThreadPool::TryParallelFor*), although we also support scheduling
+// of asynchronous tasks (ThreadPool::Schedule), and the construction
+// of multi-loop parallel sections (ThreadPool::ParallelSection).
+//
+// This high level API is accessed via static methods on the
+// ThreadPool class. These methods map the operations onto one of
+// three low-level implementations: (#1) direct execution of the
+// operations if there is no thread pool configured, (#2) execution of
+// the operations using the modified Eigen threadpool, (#3) execution
+// of the operations using OpenMP. Option #1 enables execution in
+// simple settings without needing threads. Option #2 is the
+// preferred approach for use in settings with parallelism.
+//
+// The high-level part of the thread pool is responsible for:
+//
+// - Exposing the desired degree of parallelism to user code, and to
+// libraries such as MLAS. This lets the libraries tailor the
+// extent to which they parallelize work.
+//
+// - Handling trivial cases (such as directly running parallel loops
+// with only a single iteration, or with no iterations at all).
+//
+// - Deciding how to divide work efficiently between the threads
+// available.
+//
+// The ThreadPool::TryParallelFor methods do this based on cost
+// estimates supplied by the caller, and are designed to support
+// loops with small amounts of work per iteration. The loop body is
+// supplied as a function taking a [start,end) range of iterations
+// to execute (avoiding the need for per-iteration std::function
+// calls, or a reliance upon inlining to avoid those calls).
+//
+// ThreadPool::TrySimpleParallelFor uses a simpler single-iteration
+// API based on the assumption that the caller has divided work to
+// an appropriate granularity.
+//
+// - When used with the Eigen-based thread pool, the implementation of
+// all of the loops maps down onto
+// ThreadPool::ParallelForFixedBlockSizeScheduling. This method
+// takes the degree of parallelism (d_of_p) and work distribution
+// block size (from the cost-based heuristics), and creates a set of
+// tasks in the underlying thread pool (via
+// ThreadPool::RunInParallel).
+//
+// These tasks then run a loop which picks off batches of iterations
+// from the user's code. The distribution of these batches is
+// handled dynmamically via LoopCounter::ClaimIterations. This
+// dynamic balancing behavior helps make performance robust to any
+// variability in the execution time across iterations, and to
+// situations such as multiple loops running concurrently on the
+// same thread pool.
+//
+// - When running a series of loops inside a parallel section, the
+// LoopCounter also helps obtain affinity between these loops (i.e.,
+// iteration X of one loop will tend to run on the same thread that
+// ran iteration X of prior loops). This locality helps improve hit
+// rates in per-core caches across the series of short loops used in
+// operators like GRU.
+//
+// There are some known areas for exploration here:
+//
+// - The cost-based heuristics were developed prior to recent changes
+// to the thread pool. The heuristics seem to work well, but we
+// should revisit the tuning periodically.
+//
+// - Can we unify the APIs for the different kinds of parallel loop?
+//
+// In particular, we may be able to replace the current use of
+// TryBatchParallelFor with appropriate costs for each call site,
+// and then use TryParallelFor. This would allow for more dynamic
+// re-balancing of work between threads than the current
+// ThreadPool::PartitionWork function provides.
+//
+// - Given the extensive modifications to original Eigen code, should
+// we separate that out as a new class and remove the dependence on
+// other Eigen components.
+
+// This file use PIMPL to avoid having eigen headers here
+namespace Eigen {
+class Allocator;
+class ThreadPoolInterface;
+} // namespace Eigen
+
+namespace onnxruntime {
+
+struct TensorOpCost {
+ double bytes_loaded;
+ double bytes_stored;
+ double compute_cycles;
+};
+
+
+namespace concurrency {
+
+template <typename Environment>
+class ThreadPoolTempl;
+
+class ExtendedThreadPoolInterface;
+class LoopCounter;
+class ThreadPoolParallelSection;
+
+class ThreadPool {
+ public:
+#ifdef _WIN32
+ using NAME_CHAR_TYPE = wchar_t;
+#else
+ using NAME_CHAR_TYPE = char;
+#endif
+ // Constructs a pool for running with with "degree_of_parallelism" threads with
+ // specified "name". env->StartThread() is used to create individual threads
+ // with the given ThreadOptions. If "low_latency_hint" is true the thread pool
+ // implementation may use it as a hint that lower latency is preferred at the
+ // cost of higher CPU usage, e.g. by letting one or more idle threads spin
+ // wait. Conversely, if the threadpool is used to schedule high-latency
+ // operations like I/O the hint should be set to false.
+ //
+ // REQUIRES: degree_of_parallelism > 0
+ ThreadPool(Env* env,
+ const ThreadOptions& thread_options,
+ const NAME_CHAR_TYPE* name,
+ int degree_of_parallelism,
+ bool low_latency_hint);
+
+ // Waits until all scheduled work has finished and then destroy the
+ // set of threads.
+ ~ThreadPool();
+
+ // Start and end a multi-loop parallel section. Parallel loops can
+ // be executed directly (without using this API), but entering a
+ // parallel section allows the runtime system to amortize loop
+ // entry/exit costs over multiple loops, and allows it to promote
+ // affinity between corresponding iterations of different loops.
+ //
+ // Multi-loop sections would typically be used in cases where a
+ // series of loops executes without much code in between them, and
+ // where it is impractical to refactor code into a single loop. For
+ // instance:
+ //
+ // {
+ // onnxruntime::concurrency::ThreadPoool::ParallelSection ps(tp);
+ // for (int x = 0; x < seq_len; x++) {
+ // TrySimpleParallelFor(tp, 16, [&]() { ... });
+ // }
+ // }
+ //
+ // The parallel section is entered via the constructor of
+ // ThreadPool::ParallelSection, and exited via the destructor.
+ // Currently, thread-local state is used to track whether or not the
+ // current thread is inside a parallel section. In contrast to
+ // handling parallel section objects explicitly in user code, this
+ // approach allows code such as MLAS to operate with/without the use
+ // of parallel sections.
+ //
+ // Parallel sections are only implemented with the Eigen threadpool.
+ // They have no effect when using OpenMP.
+ //
+ // Parallel sections may not be nested, and may not be used inside
+ // parallel loops.
+
+ class ParallelSection {
+ public:
+ explicit ParallelSection(ThreadPool *tp);
+ ~ParallelSection();
+
+ private:
+ friend class ThreadPool;
+
+ // Owning reference for the underlying ThreadPoolParallelSection
+ // which implements the thread management. We use an explicit
+ // deleter here so that the definition of
+ // ThreadPoolParallelSection does not need to be available at this
+ // point to avoid a dependence on the Eigen headers.
+ std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)>
+ ps_{nullptr, [](ThreadPoolParallelSection*){}};
+#ifndef _OPENMP
+ ThreadPool *tp_;
+#endif
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ParallelSection);
+
+ // Non-owning reference to the current thread's paralel section
+ // (or nullptr outside parallel sections).
+ static thread_local ParallelSection *current_parallel_section;
+ static_assert(std::is_trivially_destructible<decltype(current_parallel_section)>::value,
+ "Per-thread state should be trivially destructible");
+ };
+
+ // Schedules fn() for execution in the pool of threads. The function may run
+ // synchronously if it cannot be enqueued. This will occur if the thread pool's
+ // degree-of-parallelism is 1, but it may also occur for implementation-dependent
+ // reasons such as if queues used for buffering work are full.
+ static void Schedule(ThreadPool* tp,
+ std::function<void()> fn) {
+ if (tp) {
+ tp->Schedule(fn);
+ } else {
+ fn();
+ }
+ }
+
+ // ParallelFor shards the "total" units of work assuming each unit of work
+ // having roughly "cost_per_unit" cost, in cycles. Each unit of work is
+ // indexed 0, 1, ..., total - 1. Each shard contains 1 or more units of work
+ // and the total cost of each shard is roughly the same.
+ //
+ // "cost_per_unit" is an estimate of the number of CPU cycles (or nanoseconds
+ // if not CPU-bound) to complete a unit of work. Overestimating creates too
+ // many shards and CPU time will be dominated by per-shard overhead, such as
+ // Context creation. Underestimating may not fully make use of the specified
+ // parallelism, and may also cause inefficiencies due to load balancing
+ // issues and stragglers.
+
+ static void TryParallelFor(ThreadPool* tp, std::ptrdiff_t total, double cost_per_unit,
+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn) {
+ TryParallelFor(tp, total, TensorOpCost{0, 0, static_cast<double>(cost_per_unit)}, fn);
+ }
+
+ static void TryParallelFor(ThreadPool* tp, std::ptrdiff_t total, const TensorOpCost& cost_per_unit,
+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn);
+
+ // Directly schedule the 'total' tasks to the underlying threadpool, without
+ // cutting them by halves
+
+ inline static void TrySimpleParallelFor(ThreadPool* tp, std::ptrdiff_t total,
+ const std::function<void(std::ptrdiff_t)>& fn) {
+#ifdef _OPENMP
+ ORT_UNUSED_PARAMETER(tp);
+#pragma omp parallel for
+ for (std::ptrdiff_t i = 0; i < total; ++i) {
+ fn(i);
+ }
+#else
+ if (tp != nullptr) {
+ tp->SimpleParallelFor(total, fn);
+ } else {
+ for (std::ptrdiff_t i = 0; i < total; ++i) {
+ // In many cases, fn can be inlined here.
+ fn(i);
+ }
+ }
+#endif
+ }
+
+ /**
+ * Tries to call the given function in parallel, with calls split into (num_batches) batches.
+ *\param num_batches If it is zero, it will be replaced to the value of DegreeOfParallelism().
+ *\param fn A std::function or STL style functor with signature of "void f(int32_t);"
+ * Pitfall: Caller should cap `num_batches` to a reasonable value based on the cost of `fn` and the value of `total`.
+ *For example, if fn is as simple as: int sum=0; fn = [&](int i){sum +=i;} and `total` is 100, then num_batches should
+ *be just 1.
+ *
+ * ```
+ **/
+ template <typename F>
+ inline static void TryBatchParallelFor(ThreadPool* tp, std::ptrdiff_t total, F&& fn, std::ptrdiff_t num_batches) {
+#ifdef _OPENMP
+ ORT_UNUSED_PARAMETER(tp);
+ ORT_UNUSED_PARAMETER(num_batches);
+#pragma omp parallel for
+ for (std::ptrdiff_t i = 0; i < total; ++i) {
+ fn(i);
+ }
+#else
+ if (tp == nullptr) {
+ for (std::ptrdiff_t i = 0; i < total; ++i) {
+ // In many cases, fn can be inlined here.
+ fn(i);
+ }
+ return;
+ }
+ if (total <= 0)
+ return;
+
+ if (total == 1) {
+ fn(0);
+ return;
+ }
+
+ if (num_batches <= 0) {
+ num_batches = std::min<ptrdiff_t>(total, DegreeOfParallelism(tp));
+ }
+
+ if (num_batches <= 1) {
+ for (int i = 0; i < total; i++) {
+ fn(i);
+ }
+ return;
+ }
+
+ tp->SimpleParallelFor(num_batches, [&](std::ptrdiff_t batch_index) {
+ auto work = PartitionWork(batch_index, num_batches, total);
+ for (std::ptrdiff_t i = work.start; i < work.end; i++) {
+ fn(i);
+ }
+ });
+#endif
+ }
+
+ struct WorkInfo {
+ std::ptrdiff_t start;
+ std::ptrdiff_t end;
+ };
+
+ /** Calculate the start and end offsets for a batch.
+ @remarks Based on MlasPartitionWork
+ */
+ static WorkInfo PartitionWork(std::ptrdiff_t batch_idx, std::ptrdiff_t num_batches, std::ptrdiff_t total_work) {
+ const std::ptrdiff_t work_per_batch = total_work / num_batches;
+ const std::ptrdiff_t work_per_batch_extra = total_work % num_batches;
+
+ WorkInfo info;
+ if (batch_idx < work_per_batch_extra) {
+ info.start = (work_per_batch + 1) * batch_idx;
+ info.end = info.start + work_per_batch + 1;
+ } else {
+ info.start = work_per_batch * batch_idx + work_per_batch_extra;
+ info.end = info.start + work_per_batch;
+ }
+
+ return info;
+ }
+
+ //......................................................................
+ //
+ // The following static methods take into account whether OpenMP is
+ // enabled/disabled, and if the thread pool pointer is nullptr
+ // during sequential execution.
+
+ // Provide a hint to the caller for whether or not to parallelize
+ // work. This lets a caller switch to a sequential version of an
+ // algorithm rather than using calls via the ParallelFor functions.
+
+ static bool ShouldParallelize(const ThreadPool* tp);
+
+ // Return the degree of parallelism that code should assume when using the thread pool.
+ // It decouples the degree of parallelism for use with the thread pool from
+ // the implementation choice of whether this matches the number of threads created in
+ // the pool.
+ //
+ // Currently, a loop with degree-of-parallelism N is supported by a pool of N-1 threads
+ // working in combination with the thread initiating the loop.
+ static int DegreeOfParallelism(const ThreadPool* tp);
+
+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(ThreadPool);
+
+ private:
+ friend class LoopCounter;
+
+ // Returns the number of threads created in the pool. This may be different from the
+ // value returned by DegreeOfParallelism to code using the pool.
+ int NumThreads() const;
+
+ // Returns current thread id between 0 and NumThreads() - 1, if called from a
+ // thread in the pool. Returns -1 otherwise.
+ int CurrentThreadId() const;
+
+ // Run fn with up to n degree-of-parallelism enlisting the thread pool for
+ // help. The degree-of-parallelism includes the caller, and so if n==1
+ // then the function will run directly in the caller. The fork-join
+ // synchronization is handled in the thread pool, and so any state captured
+ // by fn() is safe from concurrent access once RunWithHelp returns.
+ void RunInParallel(std::function<void(unsigned idx)> fn, unsigned n);
+
+ // Divides the work represented by the range [0, total) into k shards.
+ // Calls fn(i*block_size, (i+1)*block_size) from the ith shard (0 <= i < k).
+ // Each shard may be executed on a different thread in parallel, depending on
+ // the number of threads available in the pool.
+ // When (i+1)*block_size > total, fn(i*block_size, total) is called instead.
+ // Requires 0 < block_size <= total.
+ void ParallelForFixedBlockSizeScheduling(std::ptrdiff_t total, std::ptrdiff_t block_size,
+ const std::function<void(std::ptrdiff_t, std::ptrdiff_t)>& fn);
+
+ // Return whether or not the calling thread should run a loop of
+ // num_iterations divided in chunks of block_size in parallel. If not,
+ // the caller should run the loop sequentially.
+ bool ShouldParallelizeLoop(const std::ptrdiff_t num_iterations,
+ const std::ptrdiff_t block_size = 1) const;
+
+ // Internal (non-static) parallel loop methods. Unlike the public static methods,
+ // these will not handle the cases of OpenMP builds. or builds without a threadpool.
+ void ParallelFor(std::ptrdiff_t total, double cost_per_unit,
+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn);
+
+ void ParallelFor(std::ptrdiff_t total, const TensorOpCost& cost_per_unit,
+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t)>& fn);
+
+ void SimpleParallelFor(std::ptrdiff_t total, const std::function<void(std::ptrdiff_t)>& fn);
+
+ void Schedule(std::function<void()> fn);
+
+ ThreadOptions thread_options_;
+
+ // If a thread pool is created with degree_of_parallelism != 1 then an underlying
+ // EigenThreadPool is used to create OS threads and handle work distribution to them.
+ // If degree_of_parallelism == 1 then underlying_threadpool_ is left as nullptr
+ // and parallel work is run directly by the caller.
+ ExtendedThreadPoolInterface* underlying_threadpool_ = nullptr;
+
+ // If used, underlying_threadpool_ is instantiated and owned by the ThreadPool.
+ std::unique_ptr<ThreadPoolTempl<Env> > extended_eigen_threadpool_;
+};
+
+} // namespace concurrency
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/platform/tracing.h b/include/onnxruntime/core/platform/tracing.h
new file mode 100644
index 0000000..fb61632
--- /dev/null
+++ b/include/onnxruntime/core/platform/tracing.h
@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <windows.h>
+#include <TraceLoggingProvider.h>
+
+TRACELOGGING_DECLARE_PROVIDER(telemetry_provider_handle);
diff --git a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h
new file mode 100644
index 0000000..2ee92a8
--- /dev/null
+++ b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h
@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/* ++
+Module Name:
+ TraceLoggingConfig.h
+Abstract:
+ Macro definitions used by this project's TraceLogging ETW providers:
+ - Configuration macros that select the ETW Provider Groups to be used by
+ this project.
+ - Constants for tags that are commonly used in Microsoft's
+ TraceLogging-based ETW.
+ Different versions of this file use different definitions for the
+ TraceLoggingOption configuration macros. The definitions in this file are
+ empty. As a result, providers using this configuration file will not join
+ any ETW Provider Groups and will not be given any special treatment by
+ group-sensitive ETW listeners.
+Environment:
+ User mode or kernel mode.
+--*/
+
+#pragma once
+
+// Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition
+// in this file configures the provider as a normal (non-telemetry) provider.
+#ifndef TraceLoggingOptionMicrosoftTelemetry
+#define TraceLoggingOptionMicrosoftTelemetry() \
+ TraceLoggingOptionGroup(0000000000, 00000, 00000, 0000, 0000, 0000, 0000, 0000, 000, 0000, 0000)
+ // Empty definition for TraceLoggingOptionMicrosoftTelemetry
+#endif
+
+// Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition
+// in this file configures the provider as a normal (non-telemetry) provider.
+#define TraceLoggingOptionWindowsCoreTelemetry() \
+ // Empty definition for TraceLoggingOptionWindowsCoreTelemetry
+
+// Event privacy tags. Use the PDT macro values for the tag parameter, e.g.:
+// TraceLoggingWrite(...,
+// TelemetryPrivacyDataTag(PDT_BrowsingHistory | PDT_ProductAndServiceUsage),
+// ...);
+#define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags")
+#define PDT_BrowsingHistory 0x0000000000000002u
+#define PDT_DeviceConnectivityAndConfiguration 0x0000000000000800u
+#define PDT_InkingTypingAndSpeechUtterance 0x0000000000020000u
+#define PDT_ProductAndServicePerformance 0x0000000001000000u
+#define PDT_ProductAndServiceUsage 0x0000000002000000u
+#define PDT_SoftwareSetupAndInventory 0x0000000080000000u
+
+// Event categories specified via keywords, e.g.:
+// TraceLoggingWrite(...,
+// TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),
+// ...);
+#define MICROSOFT_KEYWORD_CRITICAL_DATA 0x0000800000000000 // Bit 47
+#define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46
+#define MICROSOFT_KEYWORD_TELEMETRY 0x0000200000000000 // Bit 45
+#define MICROSOFT_KEYWORD_RESERVED_44 0x0000100000000000 // Bit 44 (reserved for future assignment)
+
+// Event categories specified via event tags, e.g.:
+// TraceLoggingWrite(...,
+// TraceLoggingEventTag(MICROSOFT_EVENTTAG_REALTIME_LATENCY),
+// ...);
+#define MICROSOFT_EVENTTAG_DROP_USER_IDS 0x00008000
+#define MICROSOFT_EVENTTAG_AGGREGATE 0x00010000
+#define MICROSOFT_EVENTTAG_DROP_PII_EXCEPT_IP 0x00020000
+#define MICROSOFT_EVENTTAG_COSTDEFERRED_LATENCY 0x00040000
+#define MICROSOFT_EVENTTAG_CORE_DATA 0x00080000
+#define MICROSOFT_EVENTTAG_INJECT_XTOKEN 0x00100000
+#define MICROSOFT_EVENTTAG_REALTIME_LATENCY 0x00200000
+#define MICROSOFT_EVENTTAG_NORMAL_LATENCY 0x00400000
+#define MICROSOFT_EVENTTAG_CRITICAL_PERSISTENCE 0x00800000
+#define MICROSOFT_EVENTTAG_NORMAL_PERSISTENCE 0x01000000
+#define MICROSOFT_EVENTTAG_DROP_PII 0x02000000
+#define MICROSOFT_EVENTTAG_HASH_PII 0x04000000
+#define MICROSOFT_EVENTTAG_MARK_PII 0x08000000
+
+// Field categories specified via field tags, e.g.:
+// TraceLoggingWrite(...,
+// TraceLoggingString(szUser, "UserName", "User's name", MICROSOFT_FIELDTAG_HASH_PII),
+// ...);
+#define MICROSOFT_FIELDTAG_DROP_PII 0x04000000
+#define MICROSOFT_FIELDTAG_HASH_PII 0x08000000
diff --git a/include/onnxruntime/core/platform/windows/readme.txt b/include/onnxruntime/core/platform/windows/readme.txt
new file mode 100644
index 0000000..f1a436f
--- /dev/null
+++ b/include/onnxruntime/core/platform/windows/readme.txt
@@ -0,0 +1,2 @@
+copied from minkernel/published/internal/telemetry/open_source/TraceLoggingConfig.h
+this is the official open source edition for these configuration settings
\ No newline at end of file
diff --git a/include/onnxruntime/core/providers/acl/acl_provider_factory.h b/include/onnxruntime/core/providers/acl/acl_provider_factory.h
new file mode 100644
index 0000000..0dc0ec2
--- /dev/null
+++ b/include/onnxruntime/core/providers/acl/acl_provider_factory.h
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ACL, _In_ OrtSessionOptions* options, int use_arena)
+ORT_ALL_ARGS_NONNULL;
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h
new file mode 100644
index 0000000..35276db
--- /dev/null
+++ b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena)
+ORT_ALL_ARGS_NONNULL;
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h
new file mode 100644
index 0000000..66f2589
--- /dev/null
+++ b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena)
+ORT_ALL_ARGS_NONNULL;
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h
new file mode 100644
index 0000000..81b5477
--- /dev/null
+++ b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param device_id cuda device id, starts from zero.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/dml/dml_provider_factory.h b/include/onnxruntime/core/providers/dml/dml_provider_factory.h
new file mode 100644
index 0000000..071e7f0
--- /dev/null
+++ b/include/onnxruntime/core/providers/dml/dml_provider_factory.h
@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma warning(push)
+#pragma warning(disable : 4201) // nonstandard extension used: nameless struct/union
+#include <d3d12.h>
+#pragma warning(pop)
+
+#ifdef __cplusplus
+ #include <DirectML.h>
+#else
+ struct IDMLDevice;
+ typedef struct IDMLDevice IDMLDevice;
+#endif
+
+// Windows pollutes the macro space, causing a build break in constants.h.
+#undef OPTIONAL
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Creates a DirectML Execution Provider which executes on the hardware adapter with the given device_id, also known as
+ * the adapter index. The device ID corresponds to the enumeration order of hardware adapters as given by
+ * IDXGIFactory::EnumAdapters. A device_id of 0 always corresponds to the default adapter, which is typically the
+ * primary display GPU installed on the system. A negative device_id is invalid.
+*/
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_DML, _In_ OrtSessionOptions* options, int device_id);
+
+/**
+ * Creates a DirectML Execution Provider using the given DirectML device, and which executes work on the supplied D3D12
+ * command queue. The DirectML device and D3D12 command queue must have the same parent ID3D12Device, or an error will
+ * be returned. The D3D12 command queue must be of type DIRECT or COMPUTE (see D3D12_COMMAND_LIST_TYPE). If this
+ * function succeeds, the inference session maintains a strong reference on both the dml_device and the command_queue
+ * objects.
+ * See also: DMLCreateDevice
+ * See also: ID3D12Device::CreateCommandQueue
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProviderEx_DML, _In_ OrtSessionOptions* options,
+ _In_ IDMLDevice* dml_device, _In_ ID3D12CommandQueue* cmd_queue);
+
+#ifdef __cplusplus
+}
+#endif
\ No newline at end of file
diff --git a/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.h b/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.h
new file mode 100644
index 0000000..b87780c
--- /dev/null
+++ b/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.h
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h b/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h
new file mode 100644
index 0000000..8f8219f
--- /dev/null
+++ b/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h
@@ -0,0 +1,15 @@
+// Copyright 2019 AMD AMDMIGraphX
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id);
+
+#ifdef __cplusplus
+}
+#endif
+
+
diff --git a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h
new file mode 100644
index 0000000..1da88b1
--- /dev/null
+++ b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h
@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+
+#include "onnxruntime_c_api.h"
+
+// NNAPIFlags are bool options we want to set for NNAPI EP
+// This enum is defined as bit flats, and cannot have negative value
+// To generate an uint32_t nnapi_flags for using with OrtSessionOptionsAppendExecutionProvider_Nnapi below,
+// uint32_t nnapi_flags = 0;
+// nnapi_flags |= NNAPI_FLAG_USE_FP16;
+enum NNAPIFlags {
+ NNAPI_FLAG_USE_NONE = 0x000,
+
+ // Using fp16 relaxation in NNAPI EP, this may improve perf but may also reduce precision
+ NNAPI_FLAG_USE_FP16 = 0x001,
+
+ // Use NCHW layout in NNAPI EP, this is only available after Android API level 29
+ // Please note for now, NNAPI perform worse using NCHW compare to using NHWC
+ NNAPI_FLAG_USE_NCHW = 0x002,
+
+ // Keep NNAPI_FLAG_MAX at the end of the enum definition
+ // And assign the last NNAPIFlag to it
+ NNAPI_FLAG_LAST = NNAPI_FLAG_USE_NCHW,
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Nnapi, _In_ OrtSessionOptions* options, uint32_t nnapi_flags);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.h b/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.h
new file mode 100644
index 0000000..ec574f7
--- /dev/null
+++ b/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.h
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+#include "core/session/onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/**
+ * \param device_id nuphar device id, starts from zero.
+ * \param settings_str Nuphar settings string.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Nuphar, _In_ OrtSessionOptions* options, int allow_unaligned_buffers, _In_ const char* settings_str);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h b/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h
new file mode 100644
index 0000000..7646644
--- /dev/null
+++ b/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h
@@ -0,0 +1,23 @@
+// Copyright(C) 2019 Intel Corporation
+// Licensed under the MIT License
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+struct ProviderInfo_OpenVINO {
+ virtual std::vector<std::string> GetAvailableDevices() const = 0;
+};
+
+extern "C" {
+#endif
+
+/**
+ * \param device_type openvino device type and precision. Could be any of
+ * CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_OpenVINO,
+ _In_ OrtSessionOptions* options, _In_ const char* device_type);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/providers.h b/include/onnxruntime/core/providers/providers.h
new file mode 100644
index 0000000..2cfd5ac
--- /dev/null
+++ b/include/onnxruntime/core/providers/providers.h
@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+namespace onnxruntime {
+class IExecutionProvider;
+
+struct IExecutionProviderFactory {
+ virtual ~IExecutionProviderFactory() = default;
+ virtual std::unique_ptr<IExecutionProvider> CreateProvider() = 0;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h b/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h
new file mode 100644
index 0000000..832171e
--- /dev/null
+++ b/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h
@@ -0,0 +1,14 @@
+// Copyright 2020 rock-chips.com Inc.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Rknpu,
+ _In_ OrtSessionOptions* options);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h
new file mode 100644
index 0000000..6152883
--- /dev/null
+++ b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param device_id hip device id, starts from zero.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id, size_t hip_mem_limit);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h
new file mode 100644
index 0000000..44debc9
--- /dev/null
+++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h
@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h
new file mode 100644
index 0000000..eb4765b
--- /dev/null
+++ b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h
@@ -0,0 +1,17 @@
+// Copyright (c) Xilinx Inc.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, const char *backend_type, int device_id);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/providers/winml/winml_provider_factory.h b/include/onnxruntime/core/providers/winml/winml_provider_factory.h
new file mode 100644
index 0000000..b08b42e
--- /dev/null
+++ b/include/onnxruntime/core/providers/winml/winml_provider_factory.h
@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+struct WinmlAdapterApi;
+typedef struct WinmlAdapterApi WinmlAdapterApi;
+
+ORT_EXPORT const WinmlAdapterApi* ORT_API_CALL OrtGetWinMLAdapter(_In_ const OrtApi* ort_api) NO_EXCEPTION;
\ No newline at end of file
diff --git a/include/onnxruntime/core/session/automl_data_containers.h b/include/onnxruntime/core/session/automl_data_containers.h
new file mode 100644
index 0000000..249e42a
--- /dev/null
+++ b/include/onnxruntime/core/session/automl_data_containers.h
@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// This header contains shared definitions for reading and writing
+// via C/C++ API Opaque data types that are registered within ORT
+#pragma once
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// This structure is used to initialize and read
+// OrtValue of opaque(com.microsoft.featurizers,DateTimeFeaturizer_TimePoint)
+struct DateTimeFeaturizerTimePointData {
+ int32_t year;
+ uint8_t month;
+ uint8_t day;
+ uint8_t hour;
+ uint8_t minute;
+ uint8_t second;
+ uint8_t dayOfWeek;
+ uint16_t dayOfYear;
+ uint8_t quarterOfYear;
+ uint8_t weekOfMonth;
+};
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
diff --git a/include/onnxruntime/core/session/environment.h b/include/onnxruntime/core/session/environment.h
new file mode 100644
index 0000000..5f7f6e2
--- /dev/null
+++ b/include/onnxruntime/core/session/environment.h
@@ -0,0 +1,91 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <atomic>
+#include <memory>
+#include "core/common/common.h"
+#include "core/common/status.h"
+#include "core/platform/threadpool.h"
+#include "core/common/logging/logging.h"
+#include "core/framework/allocator.h"
+
+struct OrtThreadingOptions;
+namespace onnxruntime {
+/** TODO: remove this class
+ Provides the runtime environment for onnxruntime.
+ Create one instance for the duration of execution.
+*/
+class Environment {
+ public:
+ /**
+ Create and initialize the runtime environment.
+ @param logging manager instance that will enable per session logger output using
+ session_options.session_logid as the logger id in messages.
+ If nullptr, the default LoggingManager MUST have been created previously as it will be used
+ for logging. This will use the default logger id in messages.
+ See core/common/logging/logging.h for details, and how LoggingManager::DefaultLogger works.
+ @param tp_options optional set of parameters controlling the number of intra and inter op threads for the global
+ threadpools.
+ @param create_global_thread_pools determine if this function will create the global threadpools or not.
+ */
+ static Status Create(std::unique_ptr<logging::LoggingManager> logging_manager,
+ std::unique_ptr<Environment>& environment,
+ const OrtThreadingOptions* tp_options = nullptr,
+ bool create_global_thread_pools = false);
+
+ logging::LoggingManager* GetLoggingManager() const {
+ return logging_manager_.get();
+ }
+
+ void SetLoggingManager(std::unique_ptr<onnxruntime::logging::LoggingManager> logging_manager) {
+ logging_manager_ = std::move(logging_manager);
+ }
+
+ onnxruntime::concurrency::ThreadPool* GetIntraOpThreadPool() const {
+ return intra_op_thread_pool_.get();
+ }
+
+ onnxruntime::concurrency::ThreadPool* GetInterOpThreadPool() const {
+ return inter_op_thread_pool_.get();
+ }
+
+ bool EnvCreatedWithGlobalThreadPools() const {
+ return create_global_thread_pools_;
+ }
+
+ /**
+ * Registers an allocator for sharing between multiple sessions.
+ * Return an error if an allocator with the same OrtMemoryInfo is already registered.
+ */
+ Status RegisterAllocator(AllocatorPtr allocator);
+
+ /**
+ * Creates and registers an allocator for sharing between multiple sessions.
+ * Return an error if an allocator with the same OrtMemoryInfo is already registered.
+ */
+ Status CreateAndRegisterAllocator(const OrtMemoryInfo& mem_info, const OrtArenaCfg* arena_cfg = nullptr);
+
+ /**
+ * Returns the list of registered allocators in this env.
+ */
+ const std::vector<AllocatorPtr>& GetRegisteredSharedAllocators() const {
+ return shared_allocators_;
+ }
+
+ private:
+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment);
+
+ Environment() = default;
+ Status Initialize(std::unique_ptr<logging::LoggingManager> logging_manager,
+ const OrtThreadingOptions* tp_options = nullptr,
+ bool create_global_thread_pools = false);
+
+ std::unique_ptr<logging::LoggingManager> logging_manager_;
+ std::unique_ptr<onnxruntime::concurrency::ThreadPool> intra_op_thread_pool_;
+ std::unique_ptr<onnxruntime::concurrency::ThreadPool> inter_op_thread_pool_;
+ bool create_global_thread_pools_{false};
+ std::vector<AllocatorPtr> shared_allocators_;
+};
+} // namespace onnxruntime
diff --git a/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.h b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.h
new file mode 100644
index 0000000..6efb20a
--- /dev/null
+++ b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.h
@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// Summary: The experimental Ort C++ API is a wrapper around the Ort C++ API.
+//
+// This C++ API further simplifies usage and provides support for modern C++ syntax/features
+// at the cost of some overhead (i.e. using std::string over char *).
+//
+// Where applicable, default memory allocator options are used unless explicitly set.
+//
+// Experimental components are designed as drop-in replacements of the regular API, requiring
+// minimal code modifications to use.
+//
+// Example: Ort::Session -> Ort::Experimental::Session
+//
+// NOTE: Experimental API components are subject to change based on feedback and provide no
+// guarantee of backwards compatibility in future releases.
+
+#pragma once
+#include "onnxruntime_cxx_api.h"
+
+namespace Ort {
+namespace Experimental {
+
+struct Session : Ort::Session {
+ Session(Env& env, std::basic_string<ORTCHAR_T>& model_path, SessionOptions& options)
+ : Ort::Session(env, model_path.data(), options){};
+ Session(Env& env, void* model_data, size_t model_data_length, SessionOptions& options)
+ : Ort::Session(env, model_data, model_data_length, options){};
+
+ // overloaded Run() with sensible defaults
+ std::vector<Ort::Value> Run(const std::vector<std::string>& input_names,
+ const std::vector<Ort::Value>& input_values,
+ const std::vector<std::string>& output_names,
+ const RunOptions& run_options = RunOptions());
+ void Run(const std::vector<std::string>& input_names,
+ const std::vector<Ort::Value>& input_values,
+ const std::vector<std::string>& output_names,
+ std::vector<Ort::Value>& output_values,
+ const RunOptions& run_options = RunOptions());
+
+ // convenience methods that simplify common lower-level API calls
+ std::vector<std::string> GetInputNames() const;
+ std::vector<std::string> GetOutputNames() const;
+ std::vector<std::string> GetOverridableInitializerNames() const;
+
+ // NOTE: shape dimensions may have a negative value to indicate a symbolic/unknown dimension.
+ std::vector<std::vector<int64_t> > GetInputShapes() const;
+ std::vector<std::vector<int64_t> > GetOutputShapes() const;
+ std::vector<std::vector<int64_t> > GetOverridableInitializerShapes() const;
+};
+
+struct Value : Ort::Value {
+ Value(OrtValue* p)
+ : Ort::Value(p){};
+
+ template <typename T>
+ static Ort::Value CreateTensor(T* p_data, size_t p_data_element_count, const std::vector<int64_t>& shape);
+ static Ort::Value CreateTensor(void* p_data, size_t p_data_byte_count, const std::vector<int64_t>& shape, ONNXTensorElementDataType type);
+
+ template <typename T>
+ static Ort::Value CreateTensor(const std::vector<int64_t>& shape);
+ static Ort::Value CreateTensor(const std::vector<int64_t>& shape, ONNXTensorElementDataType type);
+};
+
+}
+}
+
+#include "experimental_onnxruntime_cxx_inline.h"
diff --git a/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.h
new file mode 100644
index 0000000..075de83
--- /dev/null
+++ b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.h
@@ -0,0 +1,111 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// Do not include this file directly. Please include "experimental_onnxruntime_cxx_api.h" instead.
+//
+// These are the inline implementations of the C++ header APIs. They are in this separate file as to not clutter
+// the main C++ file with implementation details.
+namespace Ort {
+namespace Experimental {
+
+inline std::vector<Ort::Value> Session::Run(const std::vector<std::string>& input_names, const std::vector<Ort::Value>& input_values,
+ const std::vector<std::string>& output_names, const RunOptions& run_options) {
+ size_t output_names_count = GetOutputNames().size();
+ std::vector<Ort::Value> output_values;
+ for (size_t i = 0; i < output_names_count; i++) output_values.emplace_back(nullptr);
+ Run(input_names, input_values, output_names, output_values, run_options);
+ return output_values;
+}
+
+inline void Session::Run(const std::vector<std::string>& input_names, const std::vector<Ort::Value>& input_values,
+ const std::vector<std::string>& output_names, std::vector<Ort::Value>& output_values, const RunOptions& run_options) {
+ size_t input_names_count = input_names.size();
+ size_t output_names_count = output_names.size();
+ std::vector<const char*> input_names_(input_names_count, nullptr);
+ size_t i = 0;
+ for (auto it = input_names.begin(); it != input_names.end(); it++) input_names_[i++] = (*it).c_str();
+ std::vector<const char*> output_names_(output_names_count, nullptr);
+ i = 0;
+ for (auto it = output_names.begin(); it != output_names.end(); it++) output_names_[i++] = (*it).c_str();
+ Ort::Session::Run(run_options, input_names_.data(), input_values.data(), input_names_count, output_names_.data(), output_values.data(), output_names_count);
+}
+
+inline std::vector<std::string> Session::GetInputNames() const {
+ Ort::AllocatorWithDefaultOptions allocator;
+ size_t node_count = GetInputCount();
+ std::vector<std::string> out(node_count);
+ for (size_t i = 0; i < node_count; i++) {
+ char* tmp = GetInputName(i, allocator);
+ out[i] = tmp;
+ allocator.Free(tmp); // prevent memory leak
+ }
+ return out;
+}
+
+inline std::vector<std::string> Session::GetOutputNames() const {
+ Ort::AllocatorWithDefaultOptions allocator;
+ size_t node_count = GetOutputCount();
+ std::vector<std::string> out(node_count);
+ for (size_t i = 0; i < node_count; i++) {
+ char* tmp = GetOutputName(i, allocator);
+ out[i] = tmp;
+ allocator.Free(tmp); // prevent memory leak
+ }
+ return out;
+}
+
+inline std::vector<std::string> Session::GetOverridableInitializerNames() const {
+ Ort::AllocatorWithDefaultOptions allocator;
+ size_t init_count = GetOverridableInitializerCount();
+ std::vector<std::string> out(init_count);
+ for (size_t i = 0; i < init_count; i++) {
+ char* tmp = GetOverridableInitializerName(i, allocator);
+ out[i] = tmp;
+ allocator.Free(tmp); // prevent memory leak
+ }
+ return out;
+}
+
+inline std::vector<std::vector<int64_t>> Session::GetInputShapes() const {
+ size_t node_count = GetInputCount();
+ std::vector<std::vector<int64_t>> out(node_count);
+ for (size_t i = 0; i < node_count; i++) out[i] = GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
+ return out;
+}
+
+inline std::vector<std::vector<int64_t>> Session::GetOutputShapes() const {
+ size_t node_count = GetOutputCount();
+ std::vector<std::vector<int64_t>> out(node_count);
+ for (size_t i = 0; i < node_count; i++) out[i] = GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
+ return out;
+}
+
+inline std::vector<std::vector<int64_t>> Session::GetOverridableInitializerShapes() const {
+ size_t init_count = GetOverridableInitializerCount();
+ std::vector<std::vector<int64_t>> out(init_count);
+ for (size_t i = 0; i < init_count; i++) out[i] = GetOverridableInitializerTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
+ return out;
+}
+
+template <typename T>
+inline Ort::Value Value::CreateTensor(T* p_data, size_t p_data_element_count, const std::vector<int64_t>& shape) {
+ return CreateTensor(p_data, p_data_element_count * sizeof(T), shape, TypeToTensorType<T>::type);
+}
+
+inline Ort::Value Value::CreateTensor(void* p_data, size_t p_data_byte_count, const std::vector<int64_t>& shape, ONNXTensorElementDataType type) {
+ Ort::MemoryInfo info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+ return Ort::Value::CreateTensor(info, p_data, p_data_byte_count, shape.data(), shape.size(), type);
+}
+
+template <typename T>
+inline Ort::Value Value::CreateTensor(const std::vector<int64_t>& shape) {
+ return CreateTensor(shape, TypeToTensorType<T>::type);
+}
+
+inline Ort::Value Value::CreateTensor(const std::vector<int64_t>& shape, ONNXTensorElementDataType type) {
+ Ort::AllocatorWithDefaultOptions allocator;
+ return Ort::Value::CreateTensor(allocator, shape.data(), shape.size(), type);
+}
+
+}
+}
diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h
new file mode 100644
index 0000000..ab8b65c
--- /dev/null
+++ b/include/onnxruntime/core/session/onnxruntime_c_api.h
@@ -0,0 +1,1193 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+
+// This value is used in structures passed to ORT so that a newer version of ORT will still work with them
+#define ORT_API_VERSION 7
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// SAL2 Definitions
+#ifndef _WIN32
+#define _In_
+#define _In_z_
+#define _In_opt_
+#define _In_opt_z_
+#define _Out_
+#define _Outptr_
+#define _Out_opt_
+#define _Inout_
+#define _Inout_opt_
+#define _Frees_ptr_opt_
+#define _Ret_maybenull_
+#define _Ret_notnull_
+#define _Check_return_
+#define _Outptr_result_maybenull_
+#define _In_reads_(X)
+#define _Inout_updates_all_(X)
+#define _Out_writes_bytes_all_(X)
+#define _Out_writes_all_(X)
+#define _Success_(X)
+#define _Outptr_result_buffer_maybenull_(X)
+#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull))
+#else
+#include <specstrings.h>
+#define ORT_ALL_ARGS_NONNULL
+#endif
+
+#ifdef _WIN32
+// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort.
+// dllexport is not used, we use a .def file.
+#ifdef ORT_DLL_IMPORT
+#define ORT_EXPORT __declspec(dllimport)
+#else
+#define ORT_EXPORT
+#endif
+#define ORT_API_CALL _stdcall
+#define ORT_MUST_USE_RESULT
+#define ORTCHAR_T wchar_t
+#else
+#define ORT_EXPORT
+#define ORT_API_CALL
+#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result))
+#define ORTCHAR_T char
+#endif
+
+#ifndef ORT_TSTR
+#ifdef _WIN32
+#define ORT_TSTR(X) L##X
+#else
+#define ORT_TSTR(X) X
+#endif
+#endif
+
+// Any pointer marked with _In_ or _Out_, cannot be NULL.
+
+// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation
+// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that.
+// for ReleaseXXX(...) functions, they can accept NULL pointer.
+
+#ifdef __cplusplus
+// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept.
+// Such complex condition is needed because compilers set __cplusplus value differently.
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept)))
+#define NO_EXCEPTION noexcept
+#else
+#define NO_EXCEPTION throw()
+#endif
+#else
+#define NO_EXCEPTION
+#endif
+
+// Copied from TensorProto::DataType
+// Currently, Ort doesn't support complex64, complex128
+typedef enum ONNXTensorElementDataType {
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED,
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL,
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16,
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components
+ ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision
+} ONNXTensorElementDataType;
+
+// Synced with onnx TypeProto oneof
+typedef enum ONNXType {
+ ONNX_TYPE_UNKNOWN,
+ ONNX_TYPE_TENSOR,
+ ONNX_TYPE_SEQUENCE,
+ ONNX_TYPE_MAP,
+ ONNX_TYPE_OPAQUE,
+ ONNX_TYPE_SPARSETENSOR,
+} ONNXType;
+
+typedef enum OrtLoggingLevel {
+ ORT_LOGGING_LEVEL_VERBOSE,
+ ORT_LOGGING_LEVEL_INFO,
+ ORT_LOGGING_LEVEL_WARNING,
+ ORT_LOGGING_LEVEL_ERROR,
+ ORT_LOGGING_LEVEL_FATAL,
+} OrtLoggingLevel;
+
+typedef enum OrtErrorCode {
+ ORT_OK,
+ ORT_FAIL,
+ ORT_INVALID_ARGUMENT,
+ ORT_NO_SUCHFILE,
+ ORT_NO_MODEL,
+ ORT_ENGINE_ERROR,
+ ORT_RUNTIME_EXCEPTION,
+ ORT_INVALID_PROTOBUF,
+ ORT_MODEL_LOADED,
+ ORT_NOT_IMPLEMENTED,
+ ORT_INVALID_GRAPH,
+ ORT_EP_FAIL,
+} OrtErrorCode;
+
+#define ORT_RUNTIME_CLASS(X) \
+ struct Ort##X; \
+ typedef struct Ort##X Ort##X;
+
+// The actual types defined have an Ort prefix
+ORT_RUNTIME_CLASS(Env);
+ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success
+ORT_RUNTIME_CLASS(MemoryInfo);
+ORT_RUNTIME_CLASS(IoBinding);
+ORT_RUNTIME_CLASS(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
+ORT_RUNTIME_CLASS(Value);
+ORT_RUNTIME_CLASS(RunOptions);
+ORT_RUNTIME_CLASS(TypeInfo);
+ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
+ORT_RUNTIME_CLASS(SessionOptions);
+ORT_RUNTIME_CLASS(CustomOpDomain);
+ORT_RUNTIME_CLASS(MapTypeInfo);
+ORT_RUNTIME_CLASS(SequenceTypeInfo);
+ORT_RUNTIME_CLASS(ModelMetadata);
+ORT_RUNTIME_CLASS(ThreadPoolParams);
+ORT_RUNTIME_CLASS(ThreadingOptions);
+ORT_RUNTIME_CLASS(ArenaCfg);
+
+#ifdef _WIN32
+typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
+#else
+typedef OrtStatus* OrtStatusPtr;
+#endif
+
+// __VA_ARGS__ on Windows and Linux are different
+#define ORT_API(RETURN_TYPE, NAME, ...) ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
+
+#define ORT_API_STATUS(NAME, ...) \
+ ORT_EXPORT _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
+
+// XXX: Unfortunately, SAL annotations are known to not work with function pointers
+#define ORT_API2_STATUS(NAME, ...) \
+ _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
+
+// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT
+#define ORT_API_STATUS_IMPL(NAME, ...) \
+ _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
+
+#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input)
+
+// When passing in an allocator to any ORT function, be sure that the allocator object
+// is not destroyed until the last allocated object using it is freed.
+typedef struct OrtAllocator {
+ uint32_t version; // Initialize to ORT_API_VERSION
+ void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size);
+ void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p);
+ const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_);
+} OrtAllocator;
+
+typedef void(ORT_API_CALL* OrtLoggingFunction)(
+ void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location,
+ const char* message);
+
+// Set Graph optimization level.
+// Refer https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md
+// for in-depth undersrtanding of Graph Optimizations in ORT
+typedef enum GraphOptimizationLevel {
+ ORT_DISABLE_ALL = 0,
+ ORT_ENABLE_BASIC = 1,
+ ORT_ENABLE_EXTENDED = 2,
+ ORT_ENABLE_ALL = 99
+} GraphOptimizationLevel;
+
+typedef enum ExecutionMode {
+ ORT_SEQUENTIAL = 0,
+ ORT_PARALLEL = 1,
+} ExecutionMode;
+
+// Set the language projection, default is C, which means it will classify the language not in the list to C also.
+typedef enum OrtLanguageProjection {
+ ORT_PROJECTION_C = 0, // default
+ ORT_PROJECTION_CPLUSPLUS = 1,
+ ORT_PROJECTION_CSHARP = 2,
+ ORT_PROJECTION_PYTHON = 3,
+ ORT_PROJECTION_JAVA = 4,
+ ORT_PROJECTION_WINML = 5,
+ ORT_PROJECTION_NODEJS = 6,
+} OrtLanguageProjection;
+
+struct OrtKernelInfo;
+typedef struct OrtKernelInfo OrtKernelInfo;
+struct OrtKernelContext;
+typedef struct OrtKernelContext OrtKernelContext;
+struct OrtCustomOp;
+typedef struct OrtCustomOp OrtCustomOp;
+
+typedef enum OrtAllocatorType {
+ Invalid = -1,
+ OrtDeviceAllocator = 0,
+ OrtArenaAllocator = 1
+} OrtAllocatorType;
+
+/**
+ * memory types for allocator, exec provider specific types should be extended in each provider
+ * Whenever this struct is updated, please also update the MakeKey function in onnxruntime/core/framework/execution_provider.cc
+*/
+typedef enum OrtMemType {
+ OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider
+ OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED
+ OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED
+ OrtMemTypeDefault = 0, // the default allocator for execution provider
+} OrtMemType;
+
+typedef enum OrtCudnnConvAlgoSearch {
+ EXHAUSTIVE, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx
+ HEURISTIC, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7
+ DEFAULT, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM
+} OrtCudnnConvAlgoSearch;
+
+/// <summary>
+/// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA
+/// </summary>
+typedef struct OrtCUDAProviderOptions {
+ int device_id; // cuda device with id=0 as default device.
+ OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option
+ size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t.
+ int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo.
+ int do_copy_in_default_stream;
+} OrtCUDAProviderOptions;
+
+/// <summary>
+/// Options for the OpenVINO provider that are passed to SessionOptionsAppendExecutionProvider_OpenVINO
+/// </summary>
+typedef struct OrtOpenVINOProviderOptions {
+#ifdef __cplusplus
+ OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, num_of_threads{} {}
+#endif
+ const char* device_type; // CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32
+ unsigned char enable_vpu_fast_compile; // 0 = false, nonzero = true
+ const char* device_id;
+ size_t num_of_threads; // 0 uses default number of threads
+} OrtOpenVINOProviderOptions;
+
+struct OrtApi;
+typedef struct OrtApi OrtApi;
+
+struct OrtApiBase {
+ const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION
+ // nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file
+
+ const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION;
+};
+typedef struct OrtApiBase OrtApiBase;
+
+ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION;
+
+struct OrtApi {
+ /**
+* \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus
+*/
+ OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
+
+ OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
+
+ /**
+ * \param status must not be NULL
+ * \return The error message inside the `status`. Do not free the returned value.
+ */
+ const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
+
+ /**
+ * \param out Should be freed by `OrtReleaseEnv` after use
+ */
+ ORT_API2_STATUS(CreateEnv, OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
+
+ /**
+ * \param out Should be freed by `OrtReleaseEnv` after use
+ */
+ ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param,
+ OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
+
+ // Platform telemetry events are on by default since they are lightweight. You can manually turn them off.
+ ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env);
+ ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env);
+
+ // TODO: document the path separator convention? '/' vs '\'
+ // TODO: should specify the access characteristics of model_path. Is this read only during the
+ // execution of CreateSession, or does the OrtSession retain a handle to the file/directory
+ // and continue to access throughout the OrtSession lifetime?
+ // What sort of access is needed to model_path : read or read/write?
+ ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,
+ _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
+
+ ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length,
+ _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
+
+ ORT_API2_STATUS(Run, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options,
+ _In_reads_(input_len) const char* const* input_names,
+ _In_reads_(input_len) const OrtValue* const* input, size_t input_len,
+ _In_reads_(output_names_len) const char* const* output_names1, size_t output_names_len,
+ _Inout_updates_all_(output_names_len) OrtValue** output);
+
+ /**
+ * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use
+ */
+ ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options);
+
+ // Set filepath to save optimized model after graph level transformations.
+ ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options,
+ _In_ const ORTCHAR_T* optimized_model_filepath);
+
+ // create a copy of an existing OrtSessionOptions
+ ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options,
+ _Outptr_ OrtSessionOptions** out_options);
+
+ // Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model
+ // has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance.
+ // See [docs/ONNX_Runtime_Perf_Tuning.md] for more details.
+ ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode);
+
+ // Enable profiling for this session.
+ ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix);
+ ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options);
+
+ // Enable the memory pattern optimization.
+ // The idea is if the input shapes are the same, we could trace the internal memory allocation
+ // and generate a memory pattern for future request. So next time we could just do one allocation
+ // with a big chunk for all the internal memory allocation.
+ // Note: memory pattern optimization is only available when SequentialExecution enabled.
+ ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options);
+ ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options);
+
+ // Enable the memory arena on CPU
+ // Arena may pre-allocate memory for future usage.
+ // set this option to false if you don't want it.
+ ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options);
+ ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options);
+
+ // < logger id to use for session output
+ ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid);
+
+ // < applies to session load, initialization, etc
+ ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level);
+ ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level);
+
+ ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,
+ GraphOptimizationLevel graph_optimization_level);
+
+ // Sets the number of threads used to parallelize the execution within nodes
+ // A value of 0 means ORT will pick a default
+ // Note: If you've built ORT with OpenMP, this API has no effect on the number of threads used. In this case
+ // use the OpenMP env variables to configure the number of intra op num threads.
+ ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads);
+
+ // Sets the number of threads used to parallelize the execution of the graph (across nodes)
+ // If sequential execution is enabled this value is ignored
+ // A value of 0 means ORT will pick a default
+ ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads);
+
+ /*
+ Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain
+ */
+ ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out);
+
+ /*
+ * Add custom ops to the OrtCustomOpDomain
+ * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released
+ */
+ ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op);
+
+ /*
+ * Add a custom op domain to the OrtSessionOptions
+ * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released
+ */
+ ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain);
+
+ /*
+ * Loads a DLL named 'library_path' and looks for this entry point:
+ * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api);
+ * It then passes in the provided session options to this function along with the api base.
+ * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in
+ * session options are destroyed, or if an error occurs and it is non null.
+ */
+ ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path,
+ void** library_handle);
+
+ /**
+ * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these
+ * functions to enable them in the session:
+ * OrtSessionOptionsAppendExecutionProvider_CPU
+ * OrtSessionOptionsAppendExecutionProvider_CUDA
+ * OrtSessionOptionsAppendExecutionProvider_<remaining providers...>
+ * The order they are called indicates the preference order as well. In other words call this method
+ * on your most preferred execution provider first followed by the less preferred ones.
+ * If none are called Ort will use its internal CPU execution provider.
+ */
+
+ ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
+ ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
+ ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out);
+
+ /**
+ * \param out should be freed by OrtReleaseTypeInfo after use
+ */
+ ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info);
+
+ /**
+ * \param out should be freed by OrtReleaseTypeInfo after use
+ */
+ ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index,
+ _Outptr_ OrtTypeInfo** type_info);
+
+ /**
+ * \param out should be freed by OrtReleaseTypeInfo after use
+ */
+ ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index,
+ _Outptr_ OrtTypeInfo** type_info);
+
+ /**
+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
+ */
+ ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
+ _Outptr_ char** value);
+ ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
+ _Outptr_ char** value);
+ ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index,
+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
+
+ /**
+ * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use
+ */
+ ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out);
+
+ ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value);
+ ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value);
+ ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions*, _In_ const char* run_tag);
+
+ ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
+ ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
+ ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out);
+
+ // Set a flag so that ALL incomplete OrtRun calls that are using this instance of OrtRunOptions
+ // will exit as soon as possible.
+ ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options);
+ // Unset the terminate flag to enable this OrtRunOptions instance being used in new OrtRun calls.
+ ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options);
+
+ /**
+ * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value
+ * \param out Should be freed by calling OrtReleaseValue
+ * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx
+ */
+ ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len,
+ ONNXTensorElementDataType type, _Outptr_ OrtValue** out);
+
+ /**
+ * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after.
+ * p_data is owned by caller. OrtReleaseValue won't release p_data.
+ * \param out Should be freed by calling OrtReleaseValue
+ */
+ ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data,
+ size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type,
+ _Outptr_ OrtValue** out);
+
+ /**
+ * \Sets *out to 1 iff an OrtValue is a tensor, 0 otherwise
+ */
+ ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out);
+
+ // This function doesn't work with string tensor
+ // this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
+ ORT_API2_STATUS(GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** out);
+
+ /**
+ * \param value A tensor created from OrtCreateTensor... function.
+ * \param s each A string array. Each string in this array must be null terminated.
+ * \param s_len length of s
+ */
+ ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len);
+
+ /**
+ * \param value A tensor created from OrtCreateTensor... function.
+ * \param len total data length, not including the trailing '\0' chars.
+ */
+ ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len);
+
+ /**
+ * \param s string contents. Each string is NOT null-terminated.
+ * \param value A tensor created from OrtCreateTensor... function.
+ * \param s_len total data length, get it from OrtGetStringTensorDataLength
+ */
+ ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s,
+ size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len);
+
+ /**
+ * Don't free the 'out' value
+ */
+ ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo*,
+ _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out);
+
+ /**
+ * Return OnnxType from OrtTypeInfo
+ */
+ ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ enum ONNXType* out);
+
+ /**
+ * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo
+ */
+ ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out);
+
+ ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type);
+
+ /**
+ * \param info Created from CreateTensorTypeAndShapeInfo() function
+ * \param dim_values An array with length of `dim_count`. Its elements can contain negative values.
+ * \param dim_count length of dim_values
+ */
+ ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
+
+ ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*,
+ _Out_ enum ONNXTensorElementDataType* out);
+ ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
+ ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values,
+ size_t dim_values_length);
+ ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info,
+ _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length);
+
+ /**
+ * Return the number of elements specified by the tensor shape.
+ * Return a negative value if unknown (i.e., any dimension is negative.)
+ * e.g.
+ * [] -> 1
+ * [1,3,4] -> 12
+ * [2,0,4] -> 0
+ * [-1,3,4] -> -1
+ */
+ ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
+
+ /**
+ * \param out Should be freed by OrtReleaseTensorTypeAndShapeInfo after use
+ */
+ ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out);
+
+ /**
+ * Get the type information of an OrtValue
+ * \param value
+ * \param out The returned value should be freed by OrtReleaseTypeInfo after use
+ */
+ ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out);
+
+ ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out);
+
+ ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1,
+ enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out);
+
+ /**
+ * Convenience function for special case of CreateMemoryInfo, for the CPU allocator. Uses name = "Cpu" and id = 0.
+ */
+ ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1,
+ _Outptr_ OrtMemoryInfo** out);
+
+ /**
+ * Test if two memory info are equal
+ * \Sets 'out' to 0 if equal, -1 if not equal
+ */
+ ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out);
+
+ /**
+ * Do not free the returned value
+ */
+ ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out);
+ ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out);
+ ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out);
+ ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out);
+
+ ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outptr_ void** out);
+ ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ptr, void* p);
+ ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ const struct OrtMemoryInfo** out);
+
+ // The returned pointer doesn't have to be freed.
+ // Always returns the same instance on every invocation.
+ // Please note that this is a non-arena based allocator.
+ ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out);
+
+ // Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable
+ // optimizations that can take advantage of fixed values (such as memory planning, etc)
+ ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation,
+ _In_ int64_t dim_value);
+
+ /**
+ * APIs to support non-tensor types - map and sequence.
+ * Currently only the following types are supported
+ * Note: the following types should be kept in sync with data_types.h
+ * Map types
+ * =========
+ * std::map<std::string, std::string>
+ * std::map<std::string, int64_t>
+ * std::map<std::string, float>
+ * std::map<std::string, double>
+ * std::map<int64_t, std::string>
+ * std::map<int64_t, int64_t>
+ * std::map<int64_t, float>
+ * std::map<int64_t, double>
+ *
+ * Sequence types
+ * ==============
+ * std::vector<std::string>
+ * std::vector<int64_t>
+ * std::vector<float>
+ * std::vector<double>
+ * std::vector<std::map<std::string, float>>
+ * std::vector<std::map<int64_t, float>
+ */
+
+ /**
+ * If input OrtValue represents a map, you need to retrieve the keys and values
+ * separately. Use index=0 to retrieve keys and index=1 to retrieve values.
+ * If input OrtValue represents a sequence, use index to retrieve the index'th element
+ * of the sequence.
+ */
+ ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator,
+ _Outptr_ OrtValue** out);
+
+ /**
+ * Returns 2 for type map and N for sequence where N is the number of elements
+ * in the sequence.
+ */
+ ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out);
+
+ /**
+ * To construct a map, use num_values = 2 and 'in' should be an arrary of 2 OrtValues
+ * representing keys and values.
+ * To construct a sequence, use num_values = N where N is the number of the elements in the
+ * sequence. 'in' should be an arrary of N OrtValues.
+ * \value_type should be either map or sequence.
+ */
+ ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values,
+ enum ONNXType value_type, _Outptr_ OrtValue** out);
+
+ /**
+ * Construct OrtValue that contains a value of non-standard type created for
+ * experiments or while awaiting standardization. OrtValue in this case would contain
+ * an internal representation of the Opaque type. Opaque types are distinguished between
+ * each other by two strings 1) domain and 2) type name. The combination of the two
+ * must be unique, so the type representation is properly identified internally. The combination
+ * must be properly registered from within ORT at both compile/run time or by another API.
+ *
+ * To construct the OrtValue pass domain and type names, also a pointer to a data container
+ * the type of which must be know to both ORT and the client program. That data container may or may
+ * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for
+ * verification purposes.
+ *
+ * \domain_name - domain name for the Opaque type, null terminated.
+ * \type_name - type name for the Opaque type, null terminated.
+ * \data_contianer - data to populate OrtValue
+ * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
+ * data_container size internally.
+ */
+ ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name,
+ _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out);
+
+ /**
+ * Fetch data from an OrtValue that contains a value of non-standard type created for
+ * experiments or while awaiting standardization.
+ * \domain_name - domain name for the Opaque type, null terminated.
+ * \type_name - type name for the Opaque type, null terminated.
+ * \data_contianer - data to populate OrtValue
+ * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
+ * data_container size internally.
+ */
+
+ ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in,
+ _Out_ void* data_container, size_t data_container_size);
+
+ ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name,
+ _Out_ float* out);
+ ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name,
+ _Out_ int64_t* out);
+ ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out,
+ _Inout_ size_t* size);
+
+ ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
+ ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
+ ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index,
+ _Out_ const OrtValue** out);
+ ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index,
+ _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out);
+
+ ORT_CLASS_RELEASE(Env);
+ ORT_CLASS_RELEASE(Status); // nullptr for Status* indicates success
+ ORT_CLASS_RELEASE(MemoryInfo);
+ ORT_CLASS_RELEASE(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
+ ORT_CLASS_RELEASE(Value);
+ ORT_CLASS_RELEASE(RunOptions);
+ ORT_CLASS_RELEASE(TypeInfo);
+ ORT_CLASS_RELEASE(TensorTypeAndShapeInfo);
+ ORT_CLASS_RELEASE(SessionOptions);
+ ORT_CLASS_RELEASE(CustomOpDomain);
+
+ // End of Version 1 - DO NOT MODIFY ABOVE (see above text for more information)
+
+ // Version 2 - In development, feel free to add/remove/rearrange here
+
+ /**
+ * GetDenotationFromTypeInfo
+ * This api augments OrtTypeInfo to return denotations on the type.
+ * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor.
+ */
+ ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ const char** const denotation,
+ _Out_ size_t* len);
+
+ // OrtTypeInfo Casting methods
+
+ /**
+ * CastTypeInfoToMapTypeInfo
+ * This api augments OrtTypeInfo to return an OrtMapTypeInfo when the type is a map.
+ * The OrtMapTypeInfo has additional information about the map's key type and value type.
+ * This is used by WinML to support model reflection APIs.
+ * This is used by WinML to support model reflection APIs.
+ *
+ * Don't free the 'out' value
+ */
+ ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info,
+ _Outptr_result_maybenull_ const OrtMapTypeInfo** out);
+
+ /**
+ * CastTypeInfoToSequenceTypeInfo
+ * This api augments OrtTypeInfo to return an OrtSequenceTypeInfo when the type is a sequence.
+ * The OrtSequenceTypeInfo has additional information about the sequence's element type.
+ * This is used by WinML to support model reflection APIs.
+ *
+ * Don't free the 'out' value
+ */
+ ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info,
+ _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out);
+
+ // OrtMapTypeInfo Accessors
+
+ /**
+ * GetMapKeyType
+ * This api augments get the key type of a map. Key types are restricted to being scalar types and use ONNXTensorElementDataType.
+ * This is used by WinML to support model reflection APIs.
+ */
+ ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out);
+
+ /**
+ * GetMapValueType
+ * This api augments get the value type of a map.
+ */
+ ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info);
+
+ // OrtSequenceTypeInfo Accessors
+
+ /**
+ * GetSequenceElementType
+ * This api augments get the element type of a sequence.
+ * This is used by WinML to support model reflection APIs.
+ */
+ ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info,
+ _Outptr_ OrtTypeInfo** type_info);
+
+ ORT_CLASS_RELEASE(MapTypeInfo);
+ ORT_CLASS_RELEASE(SequenceTypeInfo);
+
+ /**
+ * \param out is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
+ * Profiling is turned ON automatically if enabled for the particular session by invoking EnableProfiling()
+ * on the SessionOptions instance used to create the session.
+ */
+ ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* sess, _Inout_ OrtAllocator* allocator, _Outptr_ char** out);
+
+ /**
+ * \param out is a pointer to the newly created object. The pointer should be freed by calling ReleaseModelMetadata after use.
+ */
+ ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* sess, _Outptr_ OrtModelMetadata** out);
+
+ /**
+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
+ */
+ ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
+ ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
+ ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator,
+ _Outptr_ char** value);
+ ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
+ /**
+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
+ * 'value' will be a nullptr if the given key is not found in the custom metadata map.
+ */
+ ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value);
+
+ ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value);
+
+ ORT_CLASS_RELEASE(ModelMetadata);
+
+ /*
+ * Creates an environment with global threadpools that will be shared across sessions.
+ * Use this in conjunction with DisablePerSessionThreads API or else the session will use
+ * its own thread pools.
+ */
+ ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel logging_level, _In_ const char* logid,
+ _In_ const OrtThreadingOptions* t_options, _Outptr_ OrtEnv** out);
+
+ /*
+ * Calling this API will make the session use the global threadpools shared across sessions.
+ * This API should be used in conjunction with CreateEnvWithGlobalThreadPools API.
+ */
+ ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options);
+
+ ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out);
+
+ ORT_CLASS_RELEASE(ThreadingOptions);
+
+ /**
+ * \param num_keys contains the number of keys in the custom metadata map
+ * \param keys is an array of null terminated strings (array count = num_keys) allocated using 'allocator'.
+ * The caller is responsible for freeing each string and the pointer array.
+ * 'keys' will be a nullptr if custom metadata map is empty.
+ */
+ ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys);
+
+ // Override symbolic dimensions (by specific name strings) with actual values
+ // if known at session initialization time to enable optimizations that can
+ // take advantage of fixed values (such as memory planning, etc)
+ ORT_API2_STATUS(AddFreeDimensionOverrideByName,
+ _Inout_ OrtSessionOptions* options, _In_ const char* dim_name,
+ _In_ int64_t dim_value);
+
+ /**
+ * \param out_ptr will hold a pointer to the array of char *
+ * representing available providers.
+ * \param provider_length is a pointer to an int variable where
+ * the number of available providers will be added.
+ * The caller is responsible for freeing each char * and the pointer
+ * array by calling ReleaseAvailableProviders().
+ */
+ ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr,
+ _In_ int* provider_length);
+
+ /**
+ * \param ptr is the pointer to an array of available providers you
+ * get after calling GetAvailableProviders().
+ * \param providers_length is the number of available providers.
+ */
+ ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr,
+ _In_ int providers_length);
+
+ /**
+ * \param value - A tensor created from OrtCreateTensor... function.
+ * \param index - index of string tensor element, length of element at index will be returned.
+ * \param out - number of UTF-8 bytes that the string contains
+ */
+ ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out);
+
+ /**
+ * \param s string element contents in UTF-8 encoding. The string is NOT null-terminated.
+ * \param value A tensor created from OrtCreateTensor... function.
+ * \param s_len element length, get it from OrtGetStringTensorElementLength.
+ * \param index offset of element of tensor to return.
+ */
+ ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s);
+
+ /**
+ * \param value - A tensor created from OrtCreateTensor... function.
+ * \param s - A null terminated UTF-8 encoded string.
+ * \param index - index of string tensor element to fill
+ */
+ ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index);
+
+ /**
+ * Set a single session configuration entry as a pair of strings
+ * If a configuration with same key exists, this will overwrite the configuration with the given config_value
+ * \param config_key A null terminated string representation of the config key
+ * \param config_value A null terminated string representation of the config value
+ * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h
+ */
+ ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options,
+ _In_z_ const char* config_key, _In_z_ const char* config_value);
+
+ /**
+ * \param sess valid OrtSession instance
+ * \param mem_info - valid OrtMemoryInfo instance
+ * \param - out a ptr to a new instance of OrtAllocator according to the spec within mem_info
+ * if successful
+ * \return OrtStatus or nullptr if successful
+ */
+ ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* sess, _In_ const OrtMemoryInfo* mem_info,
+ _Outptr_ OrtAllocator** out);
+
+ // Release instance of OrtAllocator obtained from CreateAllocator API
+ ORT_CLASS_RELEASE(Allocator);
+
+ ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr);
+
+ // Creates an IoBinding instance that allows one to bind pre-allocated OrtValues
+ // to input names. Thus if you want to use a raw on device buffer as input or output
+ // you can avoid extra copy during runtime.
+ ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* sess, _Outptr_ OrtIoBinding** out);
+
+ // Release instance or OrtIoBinding obtained from CreateIoBinding API
+ ORT_CLASS_RELEASE(IoBinding);
+
+ /**
+ * The function will bind the OrtValue to a specified input name.
+ * The OrtValue must be a Tensor. ORT would use that value in place of input for the specified name.
+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
+ * \param name - name for the model input
+ * \param val_ptr - OrtValue of Tensor type.
+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
+ */
+ ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
+
+ /**
+ * The function will bind the OrtValue to the specified output name.
+ * The OrtValue must be a Tensor. ORT would use that value in place of output for the specified name.
+ *
+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
+ * \param name - name for the model output
+ * \param val_ptr - OrtValue of Tensor type.
+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
+ */
+ ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
+
+ /**
+ * The function will bind the OrtValue to a device which specification is contained within OrtMemoryInfo
+ * You can either create an instance of OrtMemoryInfo with a device id or obtain one from the allocator that you are created/using
+ * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocated and bind a chunk of
+ * memory within OrtValue ahead of time.
+ *
+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
+ * \param name - name for the model output
+ * \param mem_info_ptr - OrtMemoryInfo
+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
+ */
+ ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* val_ptr);
+
+ /**
+ * The function returns the names of the outputs in the order they were bound. This is useful after running the model
+ * with bound outputs because the returned names are in order in which output OrtValues are returned. This API is optional
+ * to use. If you knew the order of outputs and its names you used for binding you would not need to use this API.
+ *
+ * \param binding_ptr - a ptr to an instance of OrtIoBinding created obtained from CreateIoBinding()
+ * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions()
+ * the specified allocator will be used to allocate continuous buffers for output strings and lengths.
+ * \param buffer - pointer to a continuous buffer of non-zero terminated UTF-8 encoded strings. The number of strings stored is returned count parameter.
+ * this buffer will be allocated with the specified allocator and must be freed after it is no longer needed.
+ * \param lengths - a pointer to a continuous buffer of size_t lengths of strings returned in the buffer. The number of items is returned
+ * in the count. This buffer is allocated with the specified allocator and must be freed after it is no longer needed.
+ * \para count - is the number of strings returned. If the instance of OrtIoBiding has no bound outputs, zero is returned,
+ * no memory allocation is performed and buffer and lengths are nullptr on return.
+ */
+ ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
+ _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count);
+
+ /**
+ * The function returns an array of pointers to individually allocated OrtValues that contain results of a model execution with RunWithBinding()
+ * The array contains the same number of OrtValues and they are in the same order as they were bound with BindOutput()
+ * or BindOutputToDevice().
+ * The returned OrtValues must be individually released after they are no longer needed.
+ * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after
+ * all the OrtValues contained therein are individually released.
+ *
+ * \param binding_ptr - instance of OrtIoBidning
+ * \param allocator - instance of allocator to allocate output array
+ * \param output - pointer to the allocated buffer. Returns nullptr if no outputs.
+ * \param output_count - pointer to the number of OrtValues returned. Zero if no outputs.
+ */
+ ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
+ _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count);
+
+ /** Clears any previously specified bindings for inputs/outputs
+ */
+ void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
+ void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
+
+ /**
+ * Provides element-level access into a tensor.
+ * \param location_values a pointer to an array of index values that specify an element's location in the tensor data blob
+ * \param location_values_count length of location_values
+ * \param out a pointer to the element specified by location_values
+ * e.g.
+ * Given a tensor with overall shape [3,224,224], an element at
+ * location [2,150,128] can be accessed directly.
+ *
+ * This function only works for numeric tensors.
+ * This is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
+ */
+ ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out);
+
+ /**
+ * Creates an allocator instance and registers it with the env to enable
+ * sharing between multiple sessions that use the same env instance.
+ * Lifetime of the created allocator will be valid for the duration of the environment.
+ * Returns an error if an allocator with the same OrtMemoryInfo is already registered.
+ * \param mem_info must be non-null.
+ * \param arena_cfg if nullptr defaults will be used.
+ * See docs/C_API.md for details.
+ */
+ ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info,
+ _In_ const OrtArenaCfg* arena_cfg);
+
+ /**
+ * Set the language projection for collecting telemetry data when Env is created
+ * \param projection the source projected language.
+ */
+ ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection);
+
+ /**
+ * On some platforms, this timer may not be as precise as nanoseconds
+ * For instance, on Windows and MacOS, the precision will be ~100ns
+ * \param out is set to the nanoseconds of profiling's start time
+ */
+ ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* sess, _Outptr_ uint64_t* out);
+
+ /**
+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
+ * A value of 0 means ORT will pick the default.
+ * A value of 1 means the invoking thread will be used; no threads will be created in the thread pool.
+ */
+ ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads);
+ ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads);
+
+ /**
+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
+ * Allow spinning of thread pools when their queues are empty. This API will set the value for both
+ * inter_op and intra_op threadpools.
+ * \param allow_spinning valid values are 1 and 0.
+ * 1: threadpool will spin to wait for queue to become non-empty, 0: it won't spin.
+ * Prefer a value of 0 if your CPU usage is very high.
+ */
+ ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning);
+
+ /**
+ * Add a pre-allocated initializer to a session. If a model contains an initializer with a name
+ * that is same as the name passed to this API call, ORT will use this initializer instance
+ * instead of deserializing one from the model file. This is useful when you want to share
+ * the same initializer across sessions.
+ * \param name name of the initializer
+ * \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
+ * managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
+ * to which it is added.
+ */
+ ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name,
+ _In_ const OrtValue* val);
+
+ /**
+ * Creates a custom environment with global threadpools and logger that will be shared across sessions.
+ * Use this in conjunction with DisablePerSessionThreads API or else the session will use
+ * its own thread pools.
+ *
+ * \param out should be freed by `OrtReleaseEnv` after use
+ */
+ ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel logging_level,
+ _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out);
+
+ /**
+ * Append CUDA execution provider to the session options
+ * If CUDA is not available (due to a non cuda enabled build), this function will return failure.
+ */
+ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA,
+ _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options);
+
+ /**
+ * Append OpenVINO execution provider to the session options
+ * If OpenVINO is not available (due to the OpenVINO provider shared library or its dependencies not being installed), this function will fail.
+ */
+ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO,
+ _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options);
+
+ /**
+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
+ * When this API is called, flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool.
+ * Note that an alternative way not using this option at runtime is to train and export a model without denormals
+ * and that's recommended because turning this option on may hurt model accuracy.
+ */
+ ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options);
+
+ /**
+ * Use this API to create the configuration of an arena that can eventually be used to define
+ * an arena based allocator's behavior
+ * \param max_mem - use 0 to allow ORT to choose the default
+ * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
+ * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default
+ * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default
+ * \param out - a pointer to an OrtArenaCfg instance
+ * \return a nullptr in case of success or a pointer to an OrtStatus instance in case of failure
+ * See docs/C_API.md for details on what the following parameters mean and how to choose these values
+ */
+ ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes,
+ int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out);
+
+ ORT_CLASS_RELEASE(ArenaCfg);
+
+ /**
+ * Use this API to obtain the description of the graph present in the model
+ * (doc_string field of the GraphProto message within the ModelProto message).
+ * If it doesn't exist, an empty string will be returned.
+ * \param model_metadata - an instance of OrtModelMetadata
+ * \param allocator - allocator used to allocate the string that will be returned back
+ * \param value - is set to a null terminated string allocated using 'allocator'.
+ The caller is responsible for freeing it.
+ */
+ ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata,
+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
+};
+
+/*
+ * Steps to use a custom op:
+ * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops
+ * 2 Create an OrtCustomOp structure for each op and add them to the domain
+ * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options
+*/
+#define OrtCustomOpApi OrtApi
+
+/*
+ * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by
+ * the implementor of the custom op.
+*/
+struct OrtCustomOp {
+ uint32_t version; // Initialize to ORT_API_VERSION
+
+ // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below.
+ void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api,
+ _In_ const OrtKernelInfo* info);
+
+ // Returns the name of the op
+ const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op);
+
+ // Returns the type of the execution provider, return nullptr to use CPU execution provider
+ const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op);
+
+ // Returns the count and types of the input & output tensors
+ ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
+ size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op);
+ ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
+ size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op);
+
+ // Op kernel callbacks
+ void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context);
+ void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel);
+};
+
+/*
+ * END EXPERIMENTAL
+*/
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h
new file mode 100644
index 0000000..d5aa79a
--- /dev/null
+++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h
@@ -0,0 +1,633 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// Summary: The Ort C++ API is a header only wrapper around the Ort C API.
+//
+// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors
+// and automatically releasing resources in the destructors.
+//
+// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.
+// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};).
+//
+// Only move assignment between objects is allowed, there are no copy constructors. Some objects have explicit 'Clone'
+// methods for this purpose.
+
+#pragma once
+#include "onnxruntime_c_api.h"
+#include <cstddef>
+#include <array>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <vector>
+#include <utility>
+#include <type_traits>
+
+#ifdef ORT_NO_EXCEPTIONS
+#include <iostream>
+#endif
+
+namespace Ort {
+
+// All C++ methods that can fail will throw an exception of this type
+struct Exception : std::exception {
+ Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
+
+ OrtErrorCode GetOrtErrorCode() const { return code_; }
+ const char* what() const noexcept override { return message_.c_str(); }
+
+ private:
+ std::string message_;
+ OrtErrorCode code_;
+};
+
+#ifdef ORT_NO_EXCEPTIONS
+#define ORT_CXX_API_THROW(string, code) \
+ do { \
+ std::cerr << Ort::Exception(string, code) \
+ .what() \
+ << std::endl; \
+ abort(); \
+ } while (false)
+#else
+#define ORT_CXX_API_THROW(string, code) \
+ throw Ort::Exception(string, code)
+#endif
+
+// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, it's in a template so that we can define a global variable in a header and make
+// it transparent to the users of the API.
+template <typename T>
+struct Global {
+ static const OrtApi* api_;
+};
+
+// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
+
+template <typename T>
+#ifdef ORT_API_MANUAL_INIT
+const OrtApi* Global<T>::api_{};
+inline void InitApi() { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
+#else
+const OrtApi* Global<T>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION);
+#endif
+
+// This returns a reference to the OrtApi interface in use, in case someone wants to use the C API functions
+inline const OrtApi& GetApi() { return *Global<void>::api_; }
+
+// This is a C++ wrapper for GetAvailableProviders() C API and returns
+// a vector of strings representing the available execution providers.
+std::vector<std::string> GetAvailableProviders();
+
+// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
+// This can't be done in the C API since C doesn't have function overloading.
+#define ORT_DEFINE_RELEASE(NAME) \
+ inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }
+
+ORT_DEFINE_RELEASE(Allocator);
+ORT_DEFINE_RELEASE(MemoryInfo);
+ORT_DEFINE_RELEASE(CustomOpDomain);
+ORT_DEFINE_RELEASE(Env);
+ORT_DEFINE_RELEASE(RunOptions);
+ORT_DEFINE_RELEASE(Session);
+ORT_DEFINE_RELEASE(SessionOptions);
+ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
+ORT_DEFINE_RELEASE(SequenceTypeInfo);
+ORT_DEFINE_RELEASE(MapTypeInfo);
+ORT_DEFINE_RELEASE(TypeInfo);
+ORT_DEFINE_RELEASE(Value);
+ORT_DEFINE_RELEASE(ModelMetadata);
+ORT_DEFINE_RELEASE(ThreadingOptions);
+ORT_DEFINE_RELEASE(IoBinding);
+ORT_DEFINE_RELEASE(ArenaCfg);
+
+/*! \class Ort::Float16_t
+ * \brief it is a structure that represents float16 data.
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint16_t.
+ * The size of the structure should align with uint16_t and one can freely cast
+ * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data.
+ *
+ * Generally, you can feed any of your types as float16/blfoat16 data to create a tensor
+ * on top of it, providing it can form a continuous buffer with 16-bit elements with no padding.
+ * And you can also feed a array of uint16_t elements directly. For example,
+ *
+ * \code{.unparsed}
+ * uint16_t values[] = { 15360, 16384, 16896, 17408, 17664};
+ * constexpr size_t values_length = sizeof(values) / sizeof(values[0]);
+ * std::vector<int64_t> dims = {values_length}; // one dimensional example
+ * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);
+ * // Note we are passing bytes count in this api, not number of elements -> sizeof(values)
+ * auto float16_tensor = Ort::Value::CreateTensor(info, values, sizeof(values),
+ * dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16);
+ * \endcode
+ *
+ * Here is another example, a little bit more elaborate. Let's assume that you use your own float16 type and you want to use
+ * a templated version of the API above so the type is automatically set based on your type. You will need to supply an extra
+ * template specialization.
+ *
+ * \code{.unparsed}
+ * namespace yours { struct half {}; } // assume this is your type, define this:
+ * namespace Ort {
+ * template<>
+ * struct TypeToTensorType<yours::half> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; };
+ * } //namespace Ort
+ *
+ * std::vector<yours::half> values;
+ * std::vector<int64_t> dims = {values.size()}; // one dimensional example
+ * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);
+ * // Here we are passing element count -> values.size()
+ * auto float16_tensor = Ort::Value::CreateTensor<yours::half>(info, values.data(), values.size(), dims.data(), dims.size());
+ *
+ * \endcode
+ */
+struct Float16_t {
+ uint16_t value;
+ constexpr Float16_t() noexcept : value(0) {}
+ constexpr Float16_t(uint16_t v) noexcept : value(v) {}
+ constexpr operator uint16_t() const noexcept { return value; }
+ constexpr bool operator==(const Float16_t& rhs) const noexcept { return value == rhs.value; };
+ constexpr bool operator!=(const Float16_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");
+
+/*! \class Ort::BFloat16_t
+ * \brief is a structure that represents bfloat16 data.
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint16_t.
+ * The size of the structure should align with uint16_t and one can freely cast
+ * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data.
+ *
+ * See also code examples for Float16_t above.
+ */
+struct BFloat16_t {
+ uint16_t value;
+ constexpr BFloat16_t() noexcept : value(0) {}
+ constexpr BFloat16_t(uint16_t v) noexcept : value(v) {}
+ constexpr operator uint16_t() const noexcept { return value; }
+ constexpr bool operator==(const BFloat16_t& rhs) const noexcept { return value == rhs.value; };
+ constexpr bool operator!=(const BFloat16_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
+
+// This is used internally by the C++ API. This is the common base class used by the wrapper objects.
+template <typename T>
+struct Base {
+ using contained_type = T;
+
+ Base() = default;
+ Base(T* p) : p_{p} {
+ if (!p)
+ ORT_CXX_API_THROW("Allocation failure", ORT_FAIL);
+ }
+ ~Base() { OrtRelease(p_); }
+
+ operator T*() { return p_; }
+ operator const T*() const { return p_; }
+
+ T* release() {
+ T* p = p_;
+ p_ = nullptr;
+ return p;
+ }
+
+ protected:
+ Base(const Base&) = delete;
+ Base& operator=(const Base&) = delete;
+ Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
+ void operator=(Base&& v) noexcept {
+ OrtRelease(p_);
+ p_ = v.p_;
+ v.p_ = nullptr;
+ }
+
+ T* p_{};
+
+ template <typename>
+ friend struct Unowned; // This friend line is needed to keep the centos C++ compiler from giving an error
+};
+
+template <typename T>
+struct Base<const T> {
+ using contained_type = const T;
+
+ Base() = default;
+ Base(const T* p) : p_{p} {
+ if (!p)
+ ORT_CXX_API_THROW("Invalid instance ptr", ORT_INVALID_ARGUMENT);
+ }
+ ~Base() = default;
+
+ operator const T*() const { return p_; }
+
+ protected:
+ Base(const Base&) = delete;
+ Base& operator=(const Base&) = delete;
+ Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
+ void operator=(Base&& v) noexcept {
+ p_ = v.p_;
+ v.p_ = nullptr;
+ }
+
+ const T* p_{};
+};
+
+template <typename T>
+struct Unowned : T {
+ Unowned(decltype(T::p_) p) : T{p} {}
+ Unowned(Unowned&& v) : T{v.p_} {}
+ ~Unowned() { this->release(); }
+};
+
+struct AllocatorWithDefaultOptions;
+struct MemoryInfo;
+struct Env;
+struct TypeInfo;
+struct Value;
+struct ModelMetadata;
+
+struct Env : Base<OrtEnv> {
+ Env(std::nullptr_t) {}
+ Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
+ Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
+ Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);
+ Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
+ OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
+ explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
+
+ Env& EnableTelemetryEvents();
+ Env& DisableTelemetryEvents();
+
+ Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);
+
+ static const OrtApi* s_api;
+};
+
+struct CustomOpDomain : Base<OrtCustomOpDomain> {
+ explicit CustomOpDomain(std::nullptr_t) {}
+ explicit CustomOpDomain(const char* domain);
+
+ void Add(OrtCustomOp* op);
+};
+
+struct RunOptions : Base<OrtRunOptions> {
+ RunOptions(std::nullptr_t) {}
+ RunOptions();
+
+ RunOptions& SetRunLogVerbosityLevel(int);
+ int GetRunLogVerbosityLevel() const;
+
+ RunOptions& SetRunLogSeverityLevel(int);
+ int GetRunLogSeverityLevel() const;
+
+ RunOptions& SetRunTag(const char* run_tag);
+ const char* GetRunTag() const;
+
+ // terminate ALL currently executing Session::Run calls that were made using this RunOptions instance
+ RunOptions& SetTerminate();
+ // unset the terminate flag so this RunOptions instance can be used in a new Session::Run call
+ RunOptions& UnsetTerminate();
+};
+
+struct SessionOptions : Base<OrtSessionOptions> {
+ explicit SessionOptions(std::nullptr_t) {}
+ SessionOptions();
+ explicit SessionOptions(OrtSessionOptions* p) : Base<OrtSessionOptions>{p} {}
+
+ SessionOptions Clone() const;
+
+ SessionOptions& SetIntraOpNumThreads(int intra_op_num_threads);
+ SessionOptions& SetInterOpNumThreads(int inter_op_num_threads);
+ SessionOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);
+
+ SessionOptions& EnableCpuMemArena();
+ SessionOptions& DisableCpuMemArena();
+
+ SessionOptions& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);
+
+ SessionOptions& EnableProfiling(const ORTCHAR_T* profile_file_prefix);
+ SessionOptions& DisableProfiling();
+
+ SessionOptions& EnableMemPattern();
+ SessionOptions& DisableMemPattern();
+
+ SessionOptions& SetExecutionMode(ExecutionMode execution_mode);
+
+ SessionOptions& SetLogId(const char* logid);
+ SessionOptions& SetLogSeverityLevel(int level);
+
+ SessionOptions& Add(OrtCustomOpDomain* custom_op_domain);
+
+ SessionOptions& DisablePerSessionThreads();
+
+ SessionOptions& AddConfigEntry(const char* config_key, const char* config_value);
+ SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val);
+
+ SessionOptions& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);
+ SessionOptions& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);
+};
+
+struct ModelMetadata : Base<OrtModelMetadata> {
+ explicit ModelMetadata(std::nullptr_t) {}
+ explicit ModelMetadata(OrtModelMetadata* p) : Base<OrtModelMetadata>{p} {}
+
+ char* GetProducerName(OrtAllocator* allocator) const;
+ char* GetGraphName(OrtAllocator* allocator) const;
+ char* GetDomain(OrtAllocator* allocator) const;
+ char* GetDescription(OrtAllocator* allocator) const;
+ char* GetGraphDescription(OrtAllocator* allocator) const;
+ char** GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const;
+ char* LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const;
+ int64_t GetVersion() const;
+};
+
+struct Session : Base<OrtSession> {
+ explicit Session(std::nullptr_t) {}
+ Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);
+ Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);
+
+ // Run that will allocate the output values
+ std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
+ const char* const* output_names, size_t output_count);
+ // Run for when there is a list of preallocated outputs
+ void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
+ const char* const* output_names, Value* output_values, size_t output_count);
+
+ void Run(const RunOptions& run_options, const struct IoBinding&);
+
+ size_t GetInputCount() const;
+ size_t GetOutputCount() const;
+ size_t GetOverridableInitializerCount() const;
+
+ char* GetInputName(size_t index, OrtAllocator* allocator) const;
+ char* GetOutputName(size_t index, OrtAllocator* allocator) const;
+ char* GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const;
+ char* EndProfiling(OrtAllocator* allocator) const;
+ uint64_t GetProfilingStartTimeNs() const;
+ ModelMetadata GetModelMetadata() const;
+
+ TypeInfo GetInputTypeInfo(size_t index) const;
+ TypeInfo GetOutputTypeInfo(size_t index) const;
+ TypeInfo GetOverridableInitializerTypeInfo(size_t index) const;
+};
+
+struct TensorTypeAndShapeInfo : Base<OrtTensorTypeAndShapeInfo> {
+ explicit TensorTypeAndShapeInfo(std::nullptr_t) {}
+ explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : Base<OrtTensorTypeAndShapeInfo>{p} {}
+
+ ONNXTensorElementDataType GetElementType() const;
+ size_t GetElementCount() const;
+
+ size_t GetDimensionsCount() const;
+ void GetDimensions(int64_t* values, size_t values_count) const;
+ void GetSymbolicDimensions(const char** values, size_t values_count) const;
+
+ std::vector<int64_t> GetShape() const;
+};
+
+struct SequenceTypeInfo : Base<OrtSequenceTypeInfo> {
+ explicit SequenceTypeInfo(std::nullptr_t) {}
+ explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : Base<OrtSequenceTypeInfo>{p} {}
+
+ TypeInfo GetSequenceElementType() const;
+};
+
+struct MapTypeInfo : Base<OrtMapTypeInfo> {
+ explicit MapTypeInfo(std::nullptr_t) {}
+ explicit MapTypeInfo(OrtMapTypeInfo* p) : Base<OrtMapTypeInfo>{p} {}
+
+ ONNXTensorElementDataType GetMapKeyType() const;
+ TypeInfo GetMapValueType() const;
+};
+
+struct TypeInfo : Base<OrtTypeInfo> {
+ explicit TypeInfo(std::nullptr_t) {}
+ explicit TypeInfo(OrtTypeInfo* p) : Base<OrtTypeInfo>{p} {}
+
+ Unowned<TensorTypeAndShapeInfo> GetTensorTypeAndShapeInfo() const;
+ Unowned<SequenceTypeInfo> GetSequenceTypeInfo() const;
+ Unowned<MapTypeInfo> GetMapTypeInfo() const;
+
+ ONNXType GetONNXType() const;
+};
+
+struct Value : Base<OrtValue> {
+ template <typename T>
+ static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len);
+ static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,
+ ONNXTensorElementDataType type);
+ template <typename T>
+ static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
+ static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type);
+
+ static Value CreateMap(Value& keys, Value& values);
+ static Value CreateSequence(std::vector<Value>& values);
+
+ template <typename T>
+ static Value CreateOpaque(const char* domain, const char* type_name, const T&);
+
+ template <typename T>
+ void GetOpaqueData(const char* domain, const char* type_name, T&) const;
+
+ explicit Value(std::nullptr_t) {}
+ explicit Value(OrtValue* p) : Base<OrtValue>{p} {}
+ Value(Value&&) = default;
+ Value& operator=(Value&&) = default;
+
+ bool IsTensor() const;
+ size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
+ Value GetValue(int index, OrtAllocator* allocator) const;
+
+ size_t GetStringTensorDataLength() const;
+ void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;
+
+ template <typename T>
+ T* GetTensorMutableData();
+
+ template <typename T>
+ const T* GetTensorData() const;
+
+ template <typename T>
+ T& At(const std::vector<int64_t>& location);
+
+ TypeInfo GetTypeInfo() const;
+ TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const;
+
+ size_t GetStringTensorElementLength(size_t element_index) const;
+ void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;
+
+ void FillStringTensor(const char* const* s, size_t s_len);
+ void FillStringTensorElement(const char* s, size_t index);
+};
+
+// Represents native memory allocation
+struct MemoryAllocation {
+ MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);
+ ~MemoryAllocation();
+ MemoryAllocation(const MemoryAllocation&) = delete;
+ MemoryAllocation& operator=(const MemoryAllocation&) = delete;
+ MemoryAllocation(MemoryAllocation&&);
+ MemoryAllocation& operator=(MemoryAllocation&&);
+
+ void* get() { return p_; }
+ size_t size() const { return size_; }
+
+ private:
+ OrtAllocator* allocator_;
+ void* p_;
+ size_t size_;
+};
+
+struct AllocatorWithDefaultOptions {
+ AllocatorWithDefaultOptions();
+
+ operator OrtAllocator*() { return p_; }
+ operator const OrtAllocator*() const { return p_; }
+
+ void* Alloc(size_t size);
+ // The return value will own the allocation
+ MemoryAllocation GetAllocation(size_t size);
+ void Free(void* p);
+
+ const OrtMemoryInfo* GetInfo() const;
+
+ private:
+ OrtAllocator* p_{};
+};
+
+template <typename B>
+struct BaseMemoryInfo : B {
+ BaseMemoryInfo() = default;
+ explicit BaseMemoryInfo(typename B::contained_type* p) : B(p) {}
+ ~BaseMemoryInfo() = default;
+ BaseMemoryInfo(BaseMemoryInfo&&) = default;
+ BaseMemoryInfo& operator=(BaseMemoryInfo&&) = default;
+
+ std::string GetAllocatorName() const;
+ OrtAllocatorType GetAllocatorType() const;
+ int GetDeviceId() const;
+ OrtMemType GetMemoryType() const;
+ template <typename U>
+ bool operator==(const BaseMemoryInfo<U>& o) const;
+};
+
+struct UnownedMemoryInfo : BaseMemoryInfo<Base<const OrtMemoryInfo> > {
+ explicit UnownedMemoryInfo(std::nullptr_t) {}
+ explicit UnownedMemoryInfo(const OrtMemoryInfo* p) : BaseMemoryInfo(p) {}
+};
+
+struct MemoryInfo : BaseMemoryInfo<Base<OrtMemoryInfo> > {
+ static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1);
+
+ explicit MemoryInfo(std::nullptr_t) {}
+ explicit MemoryInfo(OrtMemoryInfo* p) : BaseMemoryInfo(p) {}
+ MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);
+};
+
+struct Allocator : public Base<OrtAllocator> {
+ Allocator(const Session& session, const MemoryInfo&);
+
+ void* Alloc(size_t size) const;
+ // The return value will own the allocation
+ MemoryAllocation GetAllocation(size_t size);
+ void Free(void* p) const;
+ UnownedMemoryInfo GetInfo() const;
+};
+
+struct IoBinding : public Base<OrtIoBinding> {
+ private:
+ std::vector<std::string> GetOutputNamesHelper(OrtAllocator*) const;
+ std::vector<Value> GetOutputValuesHelper(OrtAllocator*) const;
+
+ public:
+ explicit IoBinding(Session& session);
+ void BindInput(const char* name, const Value&);
+ void BindOutput(const char* name, const Value&);
+ void BindOutput(const char* name, const MemoryInfo&);
+ std::vector<std::string> GetOutputNames() const;
+ std::vector<std::string> GetOutputNames(Allocator&) const;
+ std::vector<Value> GetOutputValues() const;
+ std::vector<Value> GetOutputValues(Allocator&) const;
+ void ClearBoundInputs();
+ void ClearBoundOutputs();
+};
+
+/*! \struct Ort::ArenaCfg
+ * \brief it is a structure that represents the configuration of an arena based allocator
+ * \details Please see docs/C_API.md for details
+ */
+struct ArenaCfg : Base<OrtArenaCfg> {
+ explicit ArenaCfg(std::nullptr_t) {}
+ /**
+ * \param max_mem - use 0 to allow ORT to choose the default
+ * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
+ * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default
+ * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default
+ * \return an instance of ArenaCfg
+ * See docs/C_API.md for details on what the following parameters mean and how to choose these values
+ */
+ ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
+};
+
+//
+// Custom OPs (only needed to implement custom OPs)
+//
+
+struct CustomOpApi {
+ CustomOpApi(const OrtApi& api) : api_(api) {}
+
+ template <typename T> // T is only implemented for float, int64_t, and string
+ T KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name);
+
+ OrtTensorTypeAndShapeInfo* GetTensorTypeAndShape(_In_ const OrtValue* value);
+ size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info);
+ ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo* info);
+ size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info);
+ void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length);
+ void SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
+
+ template <typename T>
+ T* GetTensorMutableData(_Inout_ OrtValue* value);
+ template <typename T>
+ const T* GetTensorData(_Inout_ const OrtValue* value);
+
+ std::vector<int64_t> GetTensorShape(const OrtTensorTypeAndShapeInfo* info);
+ void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input);
+ size_t KernelContext_GetInputCount(const OrtKernelContext* context);
+ const OrtValue* KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index);
+ size_t KernelContext_GetOutputCount(const OrtKernelContext* context);
+ OrtValue* KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count);
+
+ void ThrowOnError(OrtStatus* result);
+
+ private:
+ const OrtApi& api_;
+};
+
+template <typename TOp, typename TKernel>
+struct CustomOpBase : OrtCustomOp {
+ CustomOpBase() {
+ OrtCustomOp::version = ORT_API_VERSION;
+ OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };
+ OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };
+
+ OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };
+
+ OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };
+ OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };
+
+ OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };
+ OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };
+
+ OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { static_cast<TKernel*>(op_kernel)->Compute(context); };
+ OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };
+ }
+
+ // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider
+ const char* GetExecutionProviderType() const { return nullptr; }
+};
+
+} // namespace Ort
+
+#include "onnxruntime_cxx_inline.h"
diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h
new file mode 100644
index 0000000..a5ce821
--- /dev/null
+++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h
@@ -0,0 +1,985 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead.
+// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead.
+//
+// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter
+// the main C++ file with implementation details.
+
+namespace Ort {
+
+inline void ThrowOnError(const OrtApi& ort, OrtStatus* status) {
+ if (status) {
+ std::string error_message = ort.GetErrorMessage(status);
+ OrtErrorCode error_code = ort.GetErrorCode(status);
+ ort.ReleaseStatus(status);
+ ORT_CXX_API_THROW(std::move(error_message), error_code);
+ }
+}
+
+inline void ThrowOnError(OrtStatus* status) {
+ ThrowOnError(GetApi(), status);
+}
+
+// This template converts a C++ type into it's ONNXTensorElementDataType
+template <typename T>
+struct TypeToTensorType;
+template <>
+struct TypeToTensorType<float> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; };
+template <>
+struct TypeToTensorType<Float16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; };
+template <>
+struct TypeToTensorType<BFloat16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; };
+template <>
+struct TypeToTensorType<double> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; };
+template <>
+struct TypeToTensorType<int8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; };
+template <>
+struct TypeToTensorType<int16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; };
+template <>
+struct TypeToTensorType<int32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; };
+template <>
+struct TypeToTensorType<int64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; };
+template <>
+struct TypeToTensorType<uint8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; };
+template <>
+struct TypeToTensorType<uint16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; };
+template <>
+struct TypeToTensorType<uint32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; };
+template <>
+struct TypeToTensorType<uint64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; };
+template <>
+struct TypeToTensorType<bool> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; };
+
+inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size)
+ : allocator_(allocator), p_(p), size_(size) {
+}
+
+inline MemoryAllocation::~MemoryAllocation() {
+ if (p_ != nullptr) {
+ // We do not throw out of destructor
+ auto ret = GetApi().AllocatorFree(allocator_, p_);
+ static_cast<void>(ret);
+ }
+}
+
+inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) : allocator_(nullptr), p_(nullptr), size_(0) {
+ *this = std::move(o);
+}
+
+inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) {
+ OrtAllocator* alloc = nullptr;
+ void* p = nullptr;
+ size_t sz = 0;
+
+ // Swap out this
+ std::swap(alloc, allocator_);
+ std::swap(p, p_);
+ std::swap(sz, size_);
+
+ // Swap with incoming
+ std::swap(allocator_, o.allocator_);
+ std::swap(p_, o.p_);
+ std::swap(size_, o.size_);
+
+ // Destroy this instance if needed
+ MemoryAllocation this_alloc(alloc, p, sz);
+ return *this;
+}
+
+inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() {
+ ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&p_));
+}
+
+inline void* AllocatorWithDefaultOptions::Alloc(size_t size) {
+ void* out;
+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));
+ return out;
+}
+
+inline MemoryAllocation Ort::AllocatorWithDefaultOptions::GetAllocation(size_t size) {
+ void* out;
+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));
+ MemoryAllocation result(p_, out, size);
+ return result;
+}
+
+inline void AllocatorWithDefaultOptions::Free(void* p) {
+ ThrowOnError(GetApi().AllocatorFree(p_, p));
+}
+
+inline const OrtMemoryInfo* AllocatorWithDefaultOptions::GetInfo() const {
+ const OrtMemoryInfo* out;
+ ThrowOnError(GetApi().AllocatorGetInfo(p_, &out));
+ return out;
+}
+
+template <typename B>
+inline std::string BaseMemoryInfo<B>::GetAllocatorName() const {
+ const char* name = nullptr;
+ ThrowOnError(GetApi().MemoryInfoGetName(*this, &name));
+ return std::string(name);
+}
+
+template <typename B>
+inline OrtAllocatorType BaseMemoryInfo<B>::GetAllocatorType() const {
+ OrtAllocatorType type;
+ ThrowOnError(GetApi().MemoryInfoGetType(*this, &type));
+ return type;
+}
+
+template <typename B>
+int BaseMemoryInfo<B>::GetDeviceId() const {
+ int id = 0;
+ ThrowOnError(GetApi().MemoryInfoGetId(*this, &id));
+ return id;
+}
+
+template <typename B>
+inline OrtMemType BaseMemoryInfo<B>::GetMemoryType() const {
+ OrtMemType type;
+ ThrowOnError(GetApi().MemoryInfoGetMemType(*this, &type));
+ return type;
+}
+
+template <typename B>
+template <typename U>
+inline bool BaseMemoryInfo<B>::operator==(const BaseMemoryInfo<U>& o) const {
+ int comp_result = 0;
+ ThrowOnError(Ort::GetApi().CompareMemoryInfo(*this, o, &comp_result));
+ return comp_result == 0;
+}
+
+inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) {
+ OrtMemoryInfo* p;
+ ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p));
+ return MemoryInfo(p);
+}
+
+inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) {
+ ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &p_));
+}
+
+inline Allocator::Allocator(const Session& sess, const MemoryInfo& mem_info) {
+ ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &p_));
+}
+
+inline void* Allocator::Alloc(size_t size) const {
+ void* out = nullptr;
+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));
+ return out;
+}
+
+inline MemoryAllocation Ort::Allocator::GetAllocation(size_t size) {
+ void* out = nullptr;
+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));
+ MemoryAllocation result(p_, out, size);
+ return result;
+}
+
+inline void Allocator::Free(void* p) const {
+ ThrowOnError(GetApi().AllocatorFree(p_, p));
+}
+
+inline UnownedMemoryInfo Allocator::GetInfo() const {
+ const OrtMemoryInfo* out = nullptr;
+ ThrowOnError(GetApi().AllocatorGetInfo(p_, &out));
+ return UnownedMemoryInfo(out);
+}
+
+inline IoBinding::IoBinding(Session& session) {
+ ThrowOnError(GetApi().CreateIoBinding(session, &p_));
+}
+
+inline void IoBinding::BindInput(const char* name, const Value& value) {
+ ThrowOnError(GetApi().BindInput(p_, name, value));
+}
+
+inline void IoBinding::BindOutput(const char* name, const Value& value) {
+ ThrowOnError(GetApi().BindOutput(p_, name, value));
+}
+
+inline void IoBinding::BindOutput(const char* name, const MemoryInfo& mem_info) {
+ ThrowOnError(GetApi().BindOutputToDevice(p_, name, mem_info));
+}
+
+inline std::vector<std::string> IoBinding::GetOutputNamesHelper(OrtAllocator* allocator) const {
+ std::vector<std::string> result;
+ auto free_fn = [allocator](void* p) { if (p) allocator->Free(allocator, p); };
+ using Ptr = std::unique_ptr<void, decltype(free_fn)>;
+
+ char* buffer = nullptr;
+ size_t* lengths = nullptr;
+ size_t count = 0;
+ ThrowOnError(GetApi().GetBoundOutputNames(p_, allocator, &buffer, &lengths, &count));
+
+ if (count == 0) {
+ return result;
+ }
+
+ Ptr buffer_g(buffer, free_fn);
+ Ptr lengths_g(lengths, free_fn);
+
+ result.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ auto sz = *lengths;
+ result.emplace_back(buffer, sz);
+ buffer += sz;
+ ++lengths;
+ }
+ return result;
+}
+
+inline std::vector<std::string> IoBinding::GetOutputNames() const {
+ AllocatorWithDefaultOptions allocator;
+ return GetOutputNamesHelper(allocator);
+}
+
+inline std::vector<std::string> IoBinding::GetOutputNames(Allocator& allocator) const {
+ return GetOutputNamesHelper(allocator);
+}
+
+inline std::vector<Value> Ort::IoBinding::GetOutputValuesHelper(OrtAllocator* allocator) const {
+ std::vector<Value> result;
+ size_t owned = 0;
+ size_t output_count = 0;
+ // Lambda to release the buffer when no longer needed and
+ // make sure that we destroy all instances on exception
+ auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) {
+ if (buffer) {
+ while (owned < output_count) {
+ auto* p = buffer + owned++;
+ GetApi().ReleaseValue(*p);
+ }
+ allocator->Free(allocator, buffer);
+ }
+ };
+ using Ptr = std::unique_ptr<OrtValue*, decltype(free_fn)>;
+
+ OrtValue** output_buffer = nullptr;
+ ThrowOnError(GetApi().GetBoundOutputValues(p_, allocator, &output_buffer, &output_count));
+ if (output_count == 0) {
+ return result;
+ }
+
+ Ptr buffer_g(output_buffer, free_fn);
+
+ result.reserve(output_count);
+ for (size_t i = 0; i < output_count; ++i) {
+ result.emplace_back(output_buffer[i]);
+ ++owned;
+ }
+ return result;
+}
+
+inline std::vector<Value> Ort::IoBinding::GetOutputValues(Allocator& allocator) const {
+ return GetOutputValuesHelper(allocator);
+}
+
+inline std::vector<Value> Ort::IoBinding::GetOutputValues() const {
+ AllocatorWithDefaultOptions allocator;
+ return GetOutputValuesHelper(allocator);
+}
+
+inline void IoBinding::ClearBoundInputs() {
+ GetApi().ClearBoundInputs(p_);
+}
+
+inline void IoBinding::ClearBoundOutputs() {
+ GetApi().ClearBoundOutputs(p_);
+}
+
+inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) {
+ ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_));
+}
+
+inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) {
+ ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_));
+ if (strcmp(logid, "onnxruntime-node") == 0) {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
+ } else {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
+ }
+}
+
+inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) {
+ ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_));
+ if (strcmp(logid, "onnxruntime-node") == 0) {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
+ } else {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
+ }
+}
+
+inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) {
+ ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_));
+ if (strcmp(logid, "onnxruntime-node") == 0) {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
+ } else {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
+ }
+}
+
+inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
+ OrtLoggingLevel logging_level, _In_ const char* logid) {
+ ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_));
+ if (strcmp(logid, "onnxruntime-node") == 0) {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));
+ } else {
+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
+ }
+}
+
+inline Env& Env::EnableTelemetryEvents() {
+ ThrowOnError(GetApi().EnableTelemetryEvents(p_));
+ return *this;
+}
+
+inline Env& Env::DisableTelemetryEvents() {
+ ThrowOnError(GetApi().DisableTelemetryEvents(p_));
+ return *this;
+}
+
+inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) {
+ ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg));
+ return *this;
+}
+
+inline CustomOpDomain::CustomOpDomain(const char* domain) {
+ ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_));
+}
+
+inline void CustomOpDomain::Add(OrtCustomOp* op) {
+ ThrowOnError(GetApi().CustomOpDomain_Add(p_, op));
+}
+
+inline RunOptions::RunOptions() {
+ ThrowOnError(GetApi().CreateRunOptions(&p_));
+}
+
+inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) {
+ ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level));
+ return *this;
+}
+
+inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) {
+ ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level));
+ return *this;
+}
+
+inline int RunOptions::GetRunLogVerbosityLevel() const {
+ int out;
+ ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out));
+ return out;
+}
+
+inline RunOptions& RunOptions::SetRunTag(const char* run_tag) {
+ ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag));
+ return *this;
+}
+
+inline const char* RunOptions::GetRunTag() const {
+ const char* out;
+ ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out));
+ return out;
+}
+
+inline RunOptions& RunOptions::SetTerminate() {
+ ThrowOnError(GetApi().RunOptionsSetTerminate(p_));
+ return *this;
+}
+
+inline RunOptions& RunOptions::UnsetTerminate() {
+ ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_));
+ return *this;
+}
+
+inline SessionOptions::SessionOptions() {
+ ThrowOnError(GetApi().CreateSessionOptions(&p_));
+}
+
+inline SessionOptions SessionOptions::Clone() const {
+ OrtSessionOptions* out;
+ ThrowOnError(GetApi().CloneSessionOptions(p_, &out));
+ return SessionOptions{out};
+}
+
+inline SessionOptions& SessionOptions::SetIntraOpNumThreads(int intra_op_num_threads) {
+ ThrowOnError(GetApi().SetIntraOpNumThreads(p_, intra_op_num_threads));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetInterOpNumThreads(int inter_op_num_threads) {
+ ThrowOnError(GetApi().SetInterOpNumThreads(p_, inter_op_num_threads));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) {
+ ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(p_, graph_optimization_level));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) {
+ ThrowOnError(GetApi().SetOptimizedModelFilePath(p_, optimized_model_filepath));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::EnableProfiling(const ORTCHAR_T* profile_file_prefix) {
+ ThrowOnError(GetApi().EnableProfiling(p_, profile_file_prefix));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::DisableProfiling() {
+ ThrowOnError(GetApi().DisableProfiling(p_));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::EnableMemPattern() {
+ ThrowOnError(GetApi().EnableMemPattern(p_));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::DisableMemPattern() {
+ ThrowOnError(GetApi().DisableMemPattern(p_));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::EnableCpuMemArena() {
+ ThrowOnError(GetApi().EnableCpuMemArena(p_));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::DisableCpuMemArena() {
+ ThrowOnError(GetApi().DisableCpuMemArena(p_));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetExecutionMode(ExecutionMode execution_mode) {
+ ThrowOnError(GetApi().SetSessionExecutionMode(p_, execution_mode));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetLogId(const char* logid) {
+ ThrowOnError(GetApi().SetSessionLogId(p_, logid));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::SetLogSeverityLevel(int level) {
+ ThrowOnError(GetApi().SetSessionLogSeverityLevel(p_, level));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::Add(OrtCustomOpDomain* custom_op_domain) {
+ ThrowOnError(GetApi().AddCustomOpDomain(p_, custom_op_domain));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::AddConfigEntry(const char* config_key, const char* config_value) {
+ ThrowOnError(GetApi().AddSessionConfigEntry(p_, config_key, config_value));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::AddInitializer(const char* name, const OrtValue* ort_val) {
+ ThrowOnError(GetApi().AddInitializer(p_, name, ort_val));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) {
+ ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(p_, &provider_options));
+ return *this;
+}
+
+inline SessionOptions& SessionOptions::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) {
+ ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(p_, &provider_options));
+ return *this;
+}
+
+inline Session::Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) {
+ ThrowOnError(GetApi().CreateSession(env, model_path, options, &p_));
+}
+
+inline Session::Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) {
+ ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &p_));
+}
+
+inline std::vector<Value> Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
+ const char* const* output_names, size_t output_names_count) {
+ std::vector<Ort::Value> output_values;
+ for (size_t i = 0; i < output_names_count; i++)
+ output_values.emplace_back(nullptr);
+ Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_names_count);
+ return output_values;
+}
+
+inline void Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
+ const char* const* output_names, Value* output_values, size_t output_count) {
+ static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
+ auto ort_input_values = reinterpret_cast<const OrtValue**>(const_cast<Value*>(input_values));
+ auto ort_output_values = reinterpret_cast<OrtValue**>(output_values);
+ ThrowOnError(GetApi().Run(p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values));
+}
+
+inline void Session::Run(const RunOptions& run_options, const IoBinding& io_binding) {
+ ThrowOnError(GetApi().RunWithBinding(p_, run_options, io_binding));
+}
+
+inline size_t Session::GetInputCount() const {
+ size_t out;
+ ThrowOnError(GetApi().SessionGetInputCount(p_, &out));
+ return out;
+}
+
+inline size_t Session::GetOutputCount() const {
+ size_t out;
+ ThrowOnError(GetApi().SessionGetOutputCount(p_, &out));
+ return out;
+}
+
+inline size_t Session::GetOverridableInitializerCount() const {
+ size_t out;
+ ThrowOnError(GetApi().SessionGetOverridableInitializerCount(p_, &out));
+ return out;
+}
+
+inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out));
+ return out;
+}
+
+inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out));
+ return out;
+}
+
+inline char* Session::GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out));
+ return out;
+}
+
+inline char* Session::EndProfiling(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out));
+ return out;
+}
+
+inline uint64_t Session::GetProfilingStartTimeNs() const {
+ uint64_t out;
+ ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(p_, &out));
+ return out;
+}
+
+inline ModelMetadata Session::GetModelMetadata() const {
+ OrtModelMetadata* out;
+ ThrowOnError(GetApi().SessionGetModelMetadata(p_, &out));
+ return ModelMetadata{out};
+}
+
+inline char* ModelMetadata::GetProducerName(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out));
+ return out;
+}
+
+inline char* ModelMetadata::GetGraphName(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out));
+ return out;
+}
+
+inline char* ModelMetadata::GetDomain(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out));
+ return out;
+}
+
+inline char* ModelMetadata::GetDescription(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out));
+ return out;
+}
+
+inline char* ModelMetadata::GetGraphDescription(OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out));
+ return out;
+}
+
+inline char* ModelMetadata::LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const {
+ char* out;
+ ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out));
+ return out;
+}
+
+inline char** ModelMetadata::GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const {
+ char** out;
+ ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys));
+ return out;
+}
+
+inline int64_t ModelMetadata::GetVersion() const {
+ int64_t out;
+ ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out));
+ return out;
+}
+
+inline TypeInfo Session::GetInputTypeInfo(size_t index) const {
+ OrtTypeInfo* out;
+ ThrowOnError(GetApi().SessionGetInputTypeInfo(p_, index, &out));
+ return TypeInfo{out};
+}
+
+inline TypeInfo Session::GetOutputTypeInfo(size_t index) const {
+ OrtTypeInfo* out;
+ ThrowOnError(GetApi().SessionGetOutputTypeInfo(p_, index, &out));
+ return TypeInfo{out};
+}
+
+inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const {
+ OrtTypeInfo* out;
+ ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(p_, index, &out));
+ return TypeInfo{out};
+}
+
+inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const {
+ ONNXTensorElementDataType out;
+ ThrowOnError(GetApi().GetTensorElementType(p_, &out));
+ return out;
+}
+
+inline size_t TensorTypeAndShapeInfo::GetElementCount() const {
+ size_t out;
+ ThrowOnError(GetApi().GetTensorShapeElementCount(p_, &out));
+ return static_cast<size_t>(out);
+}
+
+inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const {
+ size_t out;
+ ThrowOnError(GetApi().GetDimensionsCount(p_, &out));
+ return out;
+}
+
+inline void TensorTypeAndShapeInfo::GetDimensions(int64_t* values, size_t values_count) const {
+ ThrowOnError(GetApi().GetDimensions(p_, values, values_count));
+}
+
+inline void TensorTypeAndShapeInfo::GetSymbolicDimensions(const char** values, size_t values_count) const {
+ ThrowOnError(GetApi().GetSymbolicDimensions(p_, values, values_count));
+}
+
+inline std::vector<int64_t> TensorTypeAndShapeInfo::GetShape() const {
+ std::vector<int64_t> out(GetDimensionsCount(), 0);
+ GetDimensions(out.data(), out.size());
+ return out;
+}
+
+inline Unowned<TensorTypeAndShapeInfo> TypeInfo::GetTensorTypeAndShapeInfo() const {
+ const OrtTensorTypeAndShapeInfo* out;
+ ThrowOnError(GetApi().CastTypeInfoToTensorInfo(p_, &out));
+ return Unowned<TensorTypeAndShapeInfo>(const_cast<OrtTensorTypeAndShapeInfo*>(out));
+}
+
+inline Unowned<SequenceTypeInfo> TypeInfo::GetSequenceTypeInfo() const {
+ const OrtSequenceTypeInfo* out;
+ ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(p_, &out));
+ return Unowned<SequenceTypeInfo>{const_cast<OrtSequenceTypeInfo*>(out)};
+}
+
+inline TypeInfo SequenceTypeInfo::GetSequenceElementType() const {
+ OrtTypeInfo* output;
+ ThrowOnError(GetApi().GetSequenceElementType(p_, &output));
+ return TypeInfo{output};
+}
+
+inline Unowned<MapTypeInfo> TypeInfo::GetMapTypeInfo() const {
+ const OrtMapTypeInfo* out;
+ ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(p_, &out));
+ return Unowned<MapTypeInfo>{const_cast<OrtMapTypeInfo*>(out)};
+}
+
+inline ONNXTensorElementDataType MapTypeInfo::GetMapKeyType() const {
+ ONNXTensorElementDataType out;
+ ThrowOnError(GetApi().GetMapKeyType(p_, &out));
+ return out;
+}
+
+inline TypeInfo MapTypeInfo::GetMapValueType() const {
+ OrtTypeInfo* output;
+ ThrowOnError(GetApi().GetMapValueType(p_, &output));
+ return TypeInfo{output};
+}
+
+inline ONNXType TypeInfo::GetONNXType() const {
+ ONNXType out;
+ ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(p_, &out));
+ return out;
+}
+
+template <typename T>
+inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) {
+ return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType<T>::type);
+}
+
+inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,
+ ONNXTensorElementDataType type) {
+ OrtValue* out;
+ ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out));
+ return Value{out};
+}
+
+template <typename T>
+inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) {
+ return CreateTensor(allocator, shape, shape_len, TypeToTensorType<T>::type);
+}
+
+inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) {
+ OrtValue* out;
+ ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out));
+ return Value{out};
+}
+
+inline Value Value::CreateMap(Value& keys, Value& values) {
+ OrtValue* out;
+ OrtValue* inputs[2] = {keys, values};
+ ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out));
+ return Value{out};
+}
+
+inline Value Value::CreateSequence(std::vector<Value>& values) {
+ OrtValue* out;
+ std::vector<OrtValue*> values_ort{values.data(), values.data() + values.size()};
+ ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out));
+ return Value{out};
+}
+
+template <typename T>
+inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) {
+ OrtValue* out;
+ ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out));
+ return Value{out};
+}
+
+template <typename T>
+inline void Value::GetOpaqueData(const char* domain, const char* type_name, T& out) const {
+ ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, p_, &out, sizeof(T)));
+}
+
+inline bool Value::IsTensor() const {
+ int out;
+ ThrowOnError(GetApi().IsTensor(p_, &out));
+ return out != 0;
+}
+
+inline size_t Value::GetCount() const {
+ size_t out;
+ ThrowOnError(GetApi().GetValueCount(p_, &out));
+ return out;
+}
+
+inline Value Value::GetValue(int index, OrtAllocator* allocator) const {
+ OrtValue* out;
+ ThrowOnError(GetApi().GetValue(p_, index, allocator, &out));
+ return Value{out};
+}
+
+inline size_t Value::GetStringTensorDataLength() const {
+ size_t out;
+ ThrowOnError(GetApi().GetStringTensorDataLength(p_, &out));
+ return out;
+}
+
+inline size_t Value::GetStringTensorElementLength(size_t element_index) const {
+ size_t out;
+ ThrowOnError(GetApi().GetStringTensorElementLength(p_, element_index, &out));
+ return out;
+}
+
+inline void Value::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const {
+ ThrowOnError(GetApi().GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count));
+}
+
+inline void Value::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const {
+ ThrowOnError(GetApi().GetStringTensorElement(p_, buffer_length, element_index, buffer));
+}
+
+inline void Value::FillStringTensor(const char* const* s, size_t s_len) {
+ ThrowOnError(GetApi().FillStringTensor(p_, s, s_len));
+}
+
+inline void Value::FillStringTensorElement(const char* s, size_t index) {
+ ThrowOnError(GetApi().FillStringTensorElement(p_, s, index));
+}
+
+template <typename T>
+T* Value::GetTensorMutableData() {
+ T* out;
+ ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out));
+ return out;
+}
+
+template <typename T>
+const T* Value::GetTensorData() const {
+ T* out;
+ ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out));
+ return out;
+}
+
+template <typename T>
+inline T& Value::At(const std::vector<int64_t>& location) {
+ static_assert(!std::is_same<T, std::string>::value, "this api does not support std::string");
+ T* out;
+ ThrowOnError(GetApi().TensorAt(p_, location.data(), location.size(), (void**)&out));
+ return *out;
+}
+
+inline TypeInfo Value::GetTypeInfo() const {
+ OrtTypeInfo* output;
+ ThrowOnError(GetApi().GetTypeInfo(p_, &output));
+ return TypeInfo{output};
+}
+
+inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const {
+ OrtTensorTypeAndShapeInfo* output;
+ ThrowOnError(GetApi().GetTensorTypeAndShape(p_, &output));
+ return TensorTypeAndShapeInfo{output};
+}
+
+//
+// Custom OP API Inlines
+//
+inline void CustomOpApi::ThrowOnError(OrtStatus* status) {
+ Ort::ThrowOnError(api_, status);
+}
+
+template <>
+inline float CustomOpApi::KernelInfoGetAttribute<float>(_In_ const OrtKernelInfo* info, _In_ const char* name) {
+ float out;
+ ThrowOnError(api_.KernelInfoGetAttribute_float(info, name, &out));
+ return out;
+}
+
+template <>
+inline int64_t CustomOpApi::KernelInfoGetAttribute<int64_t>(_In_ const OrtKernelInfo* info, _In_ const char* name) {
+ int64_t out;
+ ThrowOnError(api_.KernelInfoGetAttribute_int64(info, name, &out));
+ return out;
+}
+
+template <>
+inline std::string CustomOpApi::KernelInfoGetAttribute<std::string>(_In_ const OrtKernelInfo* info, _In_ const char* name) {
+ size_t size = 0;
+ std::string out;
+ OrtStatus* status = api_.KernelInfoGetAttribute_string(info, name, nullptr, &size);
+
+ // The status should be ORT_INVALID_ARGUMENT because the size is insufficient to hold the string
+ if (api_.GetErrorCode(status) == ORT_INVALID_ARGUMENT) {
+ api_.ReleaseStatus(status);
+ out.resize(size);
+ ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size));
+ out.resize(size - 1); // remove the terminating character '\0'
+ } else {
+ ThrowOnError(status);
+ }
+ return out;
+}
+
+inline OrtTensorTypeAndShapeInfo* CustomOpApi::GetTensorTypeAndShape(_In_ const OrtValue* value) {
+ OrtTensorTypeAndShapeInfo* out;
+ ThrowOnError(api_.GetTensorTypeAndShape(value, &out));
+ return out;
+}
+
+inline size_t CustomOpApi::GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info) {
+ size_t out;
+ ThrowOnError(api_.GetTensorShapeElementCount(info, &out));
+ return out;
+}
+
+inline ONNXTensorElementDataType CustomOpApi::GetTensorElementType(const OrtTensorTypeAndShapeInfo* info) {
+ ONNXTensorElementDataType out;
+ ThrowOnError(api_.GetTensorElementType(info, &out));
+ return out;
+}
+
+inline size_t CustomOpApi::GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info) {
+ size_t out;
+ ThrowOnError(api_.GetDimensionsCount(info, &out));
+ return out;
+}
+
+inline void CustomOpApi::GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) {
+ ThrowOnError(api_.GetDimensions(info, dim_values, dim_values_length));
+}
+
+inline void CustomOpApi::SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count) {
+ ThrowOnError(api_.SetDimensions(info, dim_values, dim_count));
+}
+
+template <typename T>
+inline T* CustomOpApi::GetTensorMutableData(_Inout_ OrtValue* value) {
+ T* data;
+ ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast<void**>(&data)));
+ return data;
+}
+
+template <typename T>
+inline const T* CustomOpApi::GetTensorData(_Inout_ const OrtValue* value) {
+ return GetTensorMutableData<T>(const_cast<OrtValue*>(value));
+}
+
+inline std::vector<int64_t> CustomOpApi::GetTensorShape(const OrtTensorTypeAndShapeInfo* info) {
+ std::vector<int64_t> output(GetDimensionsCount(info));
+ GetDimensions(info, output.data(), output.size());
+ return output;
+}
+
+inline void CustomOpApi::ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input) {
+ api_.ReleaseTensorTypeAndShapeInfo(input);
+}
+
+inline size_t CustomOpApi::KernelContext_GetInputCount(const OrtKernelContext* context) {
+ size_t out;
+ ThrowOnError(api_.KernelContext_GetInputCount(context, &out));
+ return out;
+}
+
+inline const OrtValue* CustomOpApi::KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index) {
+ const OrtValue* out;
+ ThrowOnError(api_.KernelContext_GetInput(context, index, &out));
+ return out;
+}
+
+inline size_t CustomOpApi::KernelContext_GetOutputCount(const OrtKernelContext* context) {
+ size_t out;
+ ThrowOnError(api_.KernelContext_GetOutputCount(context, &out));
+ return out;
+}
+
+inline OrtValue* CustomOpApi::KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index,
+ _In_ const int64_t* dim_values, size_t dim_count) {
+ OrtValue* out;
+ ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out));
+ return out;
+}
+
+inline SessionOptions& SessionOptions::DisablePerSessionThreads() {
+ ThrowOnError(GetApi().DisablePerSessionThreads(p_));
+ return *this;
+}
+
+inline std::vector<std::string> GetAvailableProviders() {
+ int len;
+ char** providers;
+ const OrtApi& api = GetApi();
+ ThrowOnError(api.GetAvailableProviders(&providers, &len));
+ std::vector<std::string> available_providers(providers, providers + len);
+ ThrowOnError(api.ReleaseAvailableProviders(providers, len));
+ return available_providers;
+}
+
+SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val);
+
+} // namespace Ort
diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
new file mode 100644
index 0000000..ff247f2
--- /dev/null
+++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+/*
+ * This file defines SessionOptions Config Keys and format of the Config Values.
+ *
+ * The Naming Convention for a SessionOptions Config Key,
+ * "[Area][.[SubArea1].[SubArea2]...].[Keyname]"
+ * Such as "ep.cuda.use_arena"
+ * The Config Key cannot be empty
+ * The maximum length of the Config Key is 128
+ *
+ * The string format of a SessionOptions Config Value is defined individually for each Config.
+ * The maximum length of the Config Value is 1024
+ */
+
+// Key for disable PrePacking,
+// If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value)
+static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking";
+
+// A value of "1" means allocators registered in the env will be used. "0" means the allocators created in the session
+// will be used. Use this to override the usage of env allocators on a per session level.
+static const char* const kOrtSessionOptionsConfigUseEnvAllocators = "session.use_env_allocators";
+
+// Set to 'ORT' (case sensitive) to load an ORT format model.
+// If unset, model type will default to ONNX unless inferred from filename ('.ort' == ORT format) or bytes to be ORT
+static const char* const kOrtSessionOptionsConfigLoadModelFormat = "session.load_model_format";
+
+// Set to 'ORT' (case sensitive) to save optimized model in ORT format when SessionOptions.optimized_model_path is set.
+// If unset, format will default to ONNX unless optimized_model_filepath ends in '.ort'.
+static const char* const kOrtSessionOptionsConfigSaveModelFormat = "session.save_model_format";
+
+// If a value is "1", flush-to-zero and denormal-as-zero are applied. The default is "0".
+// When multiple sessions are created, a main thread doesn't override changes from succeeding session options,
+// but threads in session thread pools follow option changes.
+// When ORT runs with OpenMP, the same rule is applied, i.e. the first session option to flush-to-zero and
+// denormal-as-zero is only applied to global OpenMP thread pool, which doesn't support per-session thread pool.
+// Note that an alternative way not using this option at runtime is to train and export a model without denormals
+// and that's recommended because turning this option on may hurt model accuracy.
+static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.set_denormal_as_zero";