π Awesome OCR multiple programing languages toolkits based on ONNX Runtime, OpenVINO, MNN, PaddlePaddle, TensorRT and PyTorch.
onnxruntime header
rootc664cd0f4e10
diff --git a/include/onnxruntime/core/common/code_location.h b/include/onnxruntime/core/common/code_location.hnew file mode 100644index 0000000..ff6506c--- /dev/null+++ b/include/onnxruntime/core/common/code_location.h@@ -0,0 +1,57 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <sstream>+#include <string>+#include <vector>++namespace onnxruntime {+/**+ CodeLocation captures information on where in the source code a message came from.+*/+struct CodeLocation {+ /**+ @param file_path Usually the value of __FILE__+ @param line Usually the value of __LINE__+ @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__+ */+ CodeLocation(const char* file_path, const int line, const char* func)+ : file_and_path{file_path}, line_num{line}, function{func} {+ }++ /**+ @param file_path Usually the value of __FILE__+ @param line Usually the value of __LINE__+ @param func Usually the value of __PRETTY_FUNCTION__ or __FUNCTION__+ @param stacktrace Stacktrace from source of message.+ */+ CodeLocation(const char* file_path, const int line, const char* func, const std::vector<std::string>& stacktrace)+ : file_and_path{file_path}, line_num{line}, function{func}, stacktrace(stacktrace) {+ }++ std::string FileNoPath() const {+ // assuming we always have work to do, so not trying to avoid creating a new string if+ // no path was removed.+ return file_and_path.substr(file_and_path.find_last_of("/\\") + 1);+ }++ enum Format {+ kFilename,+ kFilenameAndPath+ };++ std::string ToString(Format format = Format::kFilename) const {+ std::ostringstream out;+ out << (format == Format::kFilename ? FileNoPath() : file_and_path) << ":" << line_num << " " << function;+ return out.str();+ }++ const std::string file_and_path;+ const int line_num;+ const std::string function;+ const std::vector<std::string> stacktrace;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.hnew file mode 100644index 0000000..2d9dc34--- /dev/null+++ b/include/onnxruntime/core/common/common.h@@ -0,0 +1,289 @@+/**+ * Copyright (c) 2016-present, Facebook, Inc.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+// Portions Copyright (c) Microsoft Corporation++#pragma once++#include <algorithm>+#include <cstring>+#include <climits>+#include <functional>+#include <memory>+#include <numeric>+#include <set>+#include <sstream>+#include <string>+#include <type_traits>+#include <unordered_map>+#include <vector>+#include <chrono>++#include "core/common/code_location.h"+#include "core/common/exceptions.h"+#include "core/common/make_string.h"+#include "core/common/make_unique.h"+#include "core/common/status.h"++#ifdef USE_MIMALLOC_ARENA_ALLOCATOR+#include <mimalloc.h>+#endif++#ifdef ORT_NO_EXCEPTIONS+#include <iostream>+#endif++namespace onnxruntime {++using TimePoint = std::chrono::high_resolution_clock::time_point;++// Using statements for common classes that we refer to in ONNXRuntime very often.+// TODO(Task:137) Remove 'using' statements from header files+using common::Status;++#ifdef _WIN32+#define ORT_UNUSED_PARAMETER(x) (x)+#else+#define ORT_UNUSED_PARAMETER(x) (void)(x)+#endif++#ifndef ORT_HAVE_ATTRIBUTE+#ifdef __has_attribute+#define ORT_HAVE_ATTRIBUTE(x) __has_attribute(x)+#else+#define ORT_HAVE_ATTRIBUTE(x) 0+#endif+#endif++// ORT_ATTRIBUTE_UNUSED+//+// Prevents the compiler from complaining about or optimizing away variables+// that appear unused on Linux+#if ORT_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__))+#undef ORT_ATTRIBUTE_UNUSED+#define ORT_ATTRIBUTE_UNUSED __attribute__((__unused__))+#else+#define ORT_ATTRIBUTE_UNUSED+#endif++// macro to explicitly ignore the return value from a function call so Code Analysis doesn't complain+#define ORT_IGNORE_RETURN_VALUE(fn) \+ static_cast<void>(fn)++std::vector<std::string> GetStackTrace();+// these is a helper function that gets defined by platform/Telemetry+void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,+ const char* function, uint32_t line);++// __PRETTY_FUNCTION__ isn't a macro on gcc, so use a check for _MSC_VER+// so we only define it as one for MSVC+#if (_MSC_VER && !defined(__PRETTY_FUNCTION__))+#define __PRETTY_FUNCTION__ __FUNCTION__+#endif++// Capture where a message is coming from. Use __FUNCTION__ rather than the much longer __PRETTY_FUNCTION__+#define ORT_WHERE \+ ::onnxruntime::CodeLocation(__FILE__, __LINE__, __FUNCTION__)++#define ORT_WHERE_WITH_STACK \+ ::onnxruntime::CodeLocation(__FILE__, __LINE__, __PRETTY_FUNCTION__, ::onnxruntime::GetStackTrace())++#ifdef ORT_NO_EXCEPTIONS++#define ORT_TRY if (true)+#define ORT_CATCH(x) else if (false)+#define ORT_RETHROW++// In order to ignore the catch statement when a specific exception (not ... ) is caught and referred+// in the body of the catch statements, it is necessary to wrap the body of the catch statement into+// a lambda function. otherwise the exception referred will be undefined and cause build break+#define ORT_HANDLE_EXCEPTION(func)++// TODO, consider changing the output of the error message from std::cerr to logging when the+// exceptions are disabled, since using std::cerr might increase binary size, and std::cerr output+// might not be easily accesible on some systems such as mobile++// Throw an exception with optional message.+// NOTE: The arguments get streamed into a string via ostringstream::operator<<+// DO NOT use a printf format string, as that will not work as you expect.+#define ORT_THROW(...) \+ do { \+ std::cerr << ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, \+ ::onnxruntime::MakeString(__VA_ARGS__)) \+ .what() \+ << std::endl; \+ abort(); \+ } while (false)++// Just in order to mark things as not implemented. Do not use in final code.+#define ORT_NOT_IMPLEMENTED(...) \+ do { \+ std::cerr << ::onnxruntime::NotImplementedException(::onnxruntime::MakeString(__VA_ARGS__)) \+ .what() \+ << std::endl; \+ abort(); \+ } while (false)++// Check condition.+// NOTE: The arguments get streamed into a string via ostringstream::operator<<+// DO NOT use a printf format string, as that will not work as you expect.+#define ORT_ENFORCE(condition, ...) \+ do { \+ if (!(condition)) { \+ std::cerr << ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, #condition, \+ ::onnxruntime::MakeString(__VA_ARGS__)) \+ .what() \+ << std::endl; \+ abort(); \+ } \+ } while (false)++#define ORT_THROW_EX(ex, ...) \+ do { \+ std::cerr << #ex << "(" << ::onnxruntime::MakeString(__VA_ARGS__) << ")" << std::endl; \+ abort(); \+ } while (false)++#else++#define ORT_TRY try+#define ORT_CATCH(x) catch (x)+#define ORT_RETHROW throw;++#define ORT_HANDLE_EXCEPTION(func) func()++// Throw an exception with optional message.+// NOTE: The arguments get streamed into a string via ostringstream::operator<<+// DO NOT use a printf format string, as that will not work as you expect.+#define ORT_THROW(...) \+ throw ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, ::onnxruntime::MakeString(__VA_ARGS__))++// Just in order to mark things as not implemented. Do not use in final code.+#define ORT_NOT_IMPLEMENTED(...) \+ throw ::onnxruntime::NotImplementedException(::onnxruntime::MakeString(__VA_ARGS__))++// Check condition.+// NOTE: The arguments get streamed into a string via ostringstream::operator<<+// DO NOT use a printf format string, as that will not work as you expect.+#define ORT_ENFORCE(condition, ...) \+ if (!(condition)) \+ throw ::onnxruntime::OnnxRuntimeException(ORT_WHERE_WITH_STACK, #condition, \+ ::onnxruntime::MakeString(__VA_ARGS__))++#define ORT_THROW_EX(ex, ...) \+ throw ex(__VA_ARGS__)++#endif++#define ORT_MAKE_STATUS(category, code, ...) \+ ::onnxruntime::common::Status(::onnxruntime::common::category, \+ ::onnxruntime::common::code, \+ ::onnxruntime::MakeString(__VA_ARGS__))++// Check condition. if met, return status.+#define ORT_RETURN_IF(condition, ...) \+ if (condition) { \+ return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, \+ "Satisfied, but should not be: " #condition "\n", \+ ORT_WHERE.ToString(), ::onnxruntime::MakeString(__VA_ARGS__)); \+ }++// Check condition. if not met, return status.+#define ORT_RETURN_IF_NOT(condition, ...) \+ if (!(condition)) { \+ return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Not satisfied: " #condition "\n", \+ ORT_WHERE.ToString(), ::onnxruntime::MakeString(__VA_ARGS__)); \+ }++// Macros to disable the copy and/or move ctor and assignment methods+// These are usually placed in the private: declarations for a class.++#define ORT_DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete++#define ORT_DISALLOW_ASSIGNMENT(TypeName) TypeName& operator=(const TypeName&) = delete++#define ORT_DISALLOW_COPY_AND_ASSIGNMENT(TypeName) \+ ORT_DISALLOW_COPY(TypeName); \+ ORT_DISALLOW_ASSIGNMENT(TypeName)++#define ORT_DISALLOW_MOVE(TypeName) \+ TypeName(TypeName&&) = delete; \+ TypeName& operator=(TypeName&&) = delete++#define ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(TypeName) \+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(TypeName); \+ ORT_DISALLOW_MOVE(TypeName)++#define ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id) \+ do { \+ auto _status = (expr); \+ if ((!_status.IsOK())) { \+ ::onnxruntime::LogRuntimeError(session_id, _status, __FILE__, __FUNCTION__, __LINE__); \+ return _status; \+ } \+ } while (0)++#define ORT_RETURN_IF_ERROR_SESSIONID_(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, session_id_)+#define ORT_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR_SESSIONID(expr, 0)++#define ORT_THROW_IF_ERROR(expr) \+ do { \+ auto _status = (expr); \+ if ((!_status.IsOK())) { \+ ::onnxruntime::LogRuntimeError(0, _status, __FILE__, __FUNCTION__, __LINE__); \+ ORT_THROW(_status); \+ } \+ } while (0)++// use this macro when cannot early return+#define ORT_CHECK_AND_SET_RETVAL(expr) \+ do { \+ if (retval.IsOK()) { \+ retval = (expr); \+ } \+ } while (0)++// C++ Core Guideline check suppression.+#if defined(_MSC_VER) && !defined(__NVCC__)+#define GSL_SUPPRESS(tag) [[gsl::suppress(tag)]]+#else+#define GSL_SUPPRESS(tag)+#endif++inline long long TimeDiffMicroSeconds(TimePoint start_time) {+ auto end_time = std::chrono::high_resolution_clock::now();+ return std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();+}++inline long long TimeDiffMicroSeconds(TimePoint start_time, TimePoint end_time) {+ return std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time).count();+}++struct null_type {};+inline std::string ToMBString(const std::string& s) { return s; }+#ifdef _WIN32+/**+ * Convert a wide character string into a narrow one, with local ANSI code page(like CP936)+ * DO NOT assume the result string is encoded in UTF-8+ */+std::string ToMBString(const std::wstring& s);++std::wstring ToWideString(const std::string& s);+inline std::wstring ToWideString(const std::wstring& s) { return s; }+#else+inline std::string ToWideString(const std::string& s) { return s; }+#endif++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/const_pointer_container.h b/include/onnxruntime/core/common/const_pointer_container.hnew file mode 100644index 0000000..1d821ba--- /dev/null+++ b/include/onnxruntime/core/common/const_pointer_container.h@@ -0,0 +1,85 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <type_traits>++namespace onnxruntime {+/**+ Container has T* entries. e.g. std::vector<T*>, and this class provides const access to those+ via iterators and direct access, as the standard behavior only makes the pointer constant,+ and not what is pointed too. i.e. you get a const pointer to T not a pointer to const T without this wrapper.+ See https://stackoverflow.com/questions/8017036/understanding-const-iterator-with-pointers+*/+template <typename Container>+class ConstPointerContainer {+ public:+ using T = typename std::remove_pointer<typename Container::value_type>::type;++ class ConstIterator {+ public:+ using const_iterator = typename Container::const_iterator;+ using iterator_category = std::input_iterator_tag;+ using value_type = T*;+ using difference_type = std::ptrdiff_t;+ using pointer = T**;+ using reference = T*&;++ /** Construct iterator for container that will return const T* entries.*/+ explicit ConstIterator(const_iterator position) noexcept : current_{position}, item_{nullptr} {}+ ConstIterator(const ConstIterator& other) = default;+ ConstIterator& operator=(const ConstIterator& other) = default;++ bool operator==(const ConstIterator& other) const noexcept { return current_ == other.current_; }+ bool operator!=(const ConstIterator& other) const noexcept { return current_ != other.current_; }++ ConstIterator& operator++() {+ ++current_;+ return *this;+ }++ ConstIterator operator++(int) {+ ConstIterator tmp{*this};+ ++(*this);+ return tmp;+ }++ const T*& operator*() const {+ item_ = *current_;+ return item_;+ }++ const T** operator->() const { return &(operator*()); };++ private:+ const_iterator current_;+ mutable const T* item_;+ };++ /**+ Construct wrapper class that will provide const access to the pointers in a container of non-const pointers.+ @param data Container with non-const pointers. e.g. std::vector<T*>+ */+ explicit ConstPointerContainer(const Container& data) noexcept : data_(data) {}++ size_t size() const noexcept { return data_.size(); }+ bool empty() const noexcept { return data_.empty(); }++ ConstIterator cbegin() const noexcept { return ConstIterator(data_.cbegin()); }+ ConstIterator cend() const noexcept { return ConstIterator(data_.cend()); }++ ConstIterator begin() const noexcept { return ConstIterator(data_.cbegin()); }+ ConstIterator end() const noexcept { return ConstIterator(data_.cend()); }++ const T* operator[](size_t index) const { return data_[index]; }++ const T* at(size_t index) const {+ ORT_ENFORCE(index < data_.size());+ return data_[index];+ }++ private:+ const Container& data_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/denormal.h b/include/onnxruntime/core/common/denormal.hnew file mode 100644index 0000000..baac109--- /dev/null+++ b/include/onnxruntime/core/common/denormal.h@@ -0,0 +1,17 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {++// Set or unset flush-to-zero and denormal=as-zero if SSE3 instructions are supported.+// Return true if SSE3 instruction is supported, otherwise return false.+bool SetDenormalAsZero(bool on);++#ifdef _OPENMP+// Set flush-to-zero and denormal-as-zero on OpenMP threads when on is true.+void InitializeWithDenormalAsZero(bool on);+#endif++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/eigen_common_wrapper.h b/include/onnxruntime/core/common/eigen_common_wrapper.hnew file mode 100644index 0000000..fa15399--- /dev/null+++ b/include/onnxruntime/core/common/eigen_common_wrapper.h@@ -0,0 +1,41 @@+//-----------------------------------------------------------------------------+//+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.+//+//-----------------------------------------------------------------------------+#pragma once+#include "onnxruntime_config.h"+// build/external/eigen/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h:162:71:+// error: ignoring attributes on template argument "Eigen::PacketType<const float, Eigen::DefaultDevice>::type {aka __vector(4) float}" [-Werror=ignored-attributes]+#if defined(__GNUC__)+#pragma GCC diagnostic push+#if __GNUC__ >= 6+#pragma GCC diagnostic ignored "-Wignored-attributes"+#endif+#pragma GCC diagnostic ignored "-Wunused-parameter"+#ifdef HAS_DEPRECATED_COPY+#pragma GCC diagnostic ignored "-Wdeprecated-copy"+#endif+#elif defined(_MSC_VER)+// build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76):+// warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence++// unsupported\eigen\cxx11\src\Tensor\TensorUInt128.h(150,0): Warning C4245: 'initializing': conversion from '__int64'+// to 'uint64_t', signed/unsigned mismatch+#pragma warning(push)+#pragma warning(disable : 4554)+#pragma warning(disable : 4245)+#pragma warning(disable : 4127)+#pragma warning(disable : 4805)+#pragma warning(disable : 6313)+#pragma warning(disable : 6294)+#endif++#include "unsupported/Eigen/CXX11/Tensor"++#if defined(__GNUC__)+#pragma GCC diagnostic pop+#elif defined(_MSC_VER)+#pragma warning(pop)+#endifdiff --git a/include/onnxruntime/core/common/exceptions.h b/include/onnxruntime/core/common/exceptions.hnew file mode 100644index 0000000..cbebc88--- /dev/null+++ b/include/onnxruntime/core/common/exceptions.h@@ -0,0 +1,71 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <algorithm>+#include <exception>+#include <iterator>+#include <stdexcept>+#include <string>+#include <vector>++#include "core/common/common.h"+#include "core/common/code_location.h"++namespace onnxruntime {++class NotImplementedException : public std::logic_error {+ public:+ explicit NotImplementedException(const char* _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){};+ explicit NotImplementedException(const std::string& _Message = "Function not yet implemented") noexcept : std::logic_error(_Message){};+};++class TypeMismatchException : public std::logic_error {+ public:+ TypeMismatchException() noexcept : logic_error("Type mismatch"){};+};++class OnnxRuntimeException : public std::exception {+ public:+ OnnxRuntimeException(const CodeLocation& location, const std::string& msg) noexcept+ : OnnxRuntimeException(location, nullptr, msg) {+ }++ /**+ Create a new exception that captures the location it was thrown from.+ @param location Location in the source code the exception is being thrown from+ @param failed_condition Optional string containing the condition that failed.+ e.g. "tensor.Size() == input.Size()". May be nullptr.+ @param msg Message containing additional information about the exception cause.+ */+ OnnxRuntimeException(const CodeLocation& location, const char* failed_condition, const std::string& msg)+ : location_{location} {+ std::ostringstream ss;++ ss << location.ToString(CodeLocation::kFilenameAndPath); // output full path in case just the filename is ambiguous+ if (failed_condition != nullptr) {+ ss << " " << failed_condition << " was false.";+ }++ ss << " " << msg << "\n";+ if (!location.stacktrace.empty()) {+ ss << "Stacktrace:\n";+ // skip the first entry in the stacktrace as we have that information from location.ToString()+ std::copy(++location.stacktrace.begin(), location.stacktrace.end(), std::ostream_iterator<std::string>(ss, "\n"));+ }++ what_ = ss.str();+ }++ const char* what() const noexcept override {+ return what_.c_str();+ }++ private:+ const CodeLocation location_;+ const std::vector<std::string> stacktrace_;+ std::string what_;+};++} // namespace onnxruntime\ No newline at end of filediff --git a/include/onnxruntime/core/common/logging/capture.h b/include/onnxruntime/core/common/logging/capture.hnew file mode 100644index 0000000..4f71bb3--- /dev/null+++ b/include/onnxruntime/core/common/logging/capture.h@@ -0,0 +1,115 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <cstdarg>+#include <gsl/gsl>+#include "core/common/common.h"+#include "core/common/code_location.h"+#include "core/common/logging/severity.h"++namespace onnxruntime {+namespace logging {++class Logger;+enum class DataType;++/**+ Class to capture the details of a log message.+*/+class Capture {+ public:+ /**+ Initializes a new instance of the Capture class.+ @param logger The logger.+ @param severity The severity.+ @param category The category.+ @param dataType Type of the data.+ @param location The file location the log message is coming from.+ */+ Capture(const Logger& logger, logging::Severity severity, const char* category,+ logging::DataType dataType, const CodeLocation& location)+ : logger_{&logger}, severity_{severity}, category_{category}, data_type_{dataType}, location_{location} {+ }++ /**+ The stream that can capture the message via operator<<.+ @returns Output stream.+ */+ std::ostream& Stream() noexcept {+ return stream_;+ }++#ifdef _MSC_VER+// add SAL annotation for printf format string. requires Code Analysis to run to validate usage.+#define msvc_printf_check _Printf_format_string_+#define __attribute__(x) // Disable for MSVC. Supported by GCC and CLang.+#else+#define msvc_printf_check+#endif++ /**+ Captures a printf style log message.+ @param name="format">The printf format.+ @param name="">Arguments to the printf format if needed.+ @remarks+ A maximum of 2K of output will be captured currently.+ Non-static method, so 'this' is implicit first arg, and we use format(printf(2,3)+ */+ void CapturePrintf(msvc_printf_check const char* format, ...) __attribute__((format(printf, 2, 3)));++ /**+ Process a printf style log message.+ @param format The printf format.+ @param ... Arguments to the printf format if needed.+ @remarks+ A maximum of 2K of output will be captured currently.+ Note: As va_list is 'char *', we have to disambiguate this from CapturePrintf+ so that something like "One string: %s", "the string" does not consider "the string"+ to be the va_list.+ */+ void ProcessPrintf(msvc_printf_check const char* format, va_list args);++ logging::Severity Severity() const noexcept {+ return severity_;+ }++ char SeverityPrefix() const noexcept {+ // Carefully setup so severity_ is a valid index+ GSL_SUPPRESS(bounds .2) {+ return logging::SEVERITY_PREFIX[static_cast<int>(severity_)];+ }+ }++ const char* Category() const noexcept {+ return category_;+ }++ logging::DataType DataType() const noexcept {+ return data_type_;+ }++ const CodeLocation& Location() const noexcept {+ return location_;+ }++ std::string Message() const noexcept {+ return stream_.str();+ }++ ~Capture();++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Capture);++ const Logger* logger_;+ const logging::Severity severity_;+ const char* category_;+ const logging::DataType data_type_;+ const CodeLocation location_;++ std::ostringstream stream_;+};+} // namespace logging+} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/logging/isink.h b/include/onnxruntime/core/common/logging/isink.hnew file mode 100644index 0000000..a67777d--- /dev/null+++ b/include/onnxruntime/core/common/logging/isink.h@@ -0,0 +1,41 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <string>++#include "core/common/logging/logging.h"++namespace onnxruntime {+namespace logging {+class ISink {+ public:+ ISink() = default;++ /**+ Sends the message to the sink.+ @param timestamp The timestamp.+ @param logger_id The logger identifier.+ @param message The captured message.+ */+ void Send(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) {+ SendImpl(timestamp, logger_id, message);+ }++ /**+ Sends a Profiling Event Record to the sink.+ @param Profiling Event Record+ */+ virtual void SendProfileEvent(profiling::EventRecord&) const {};++ virtual ~ISink() = default;++ private:+ // Make Code Analysis happy by disabling all for now. Enable as needed.+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ISink);++ virtual void SendImpl(const Timestamp& timestamp, const std::string& logger_id, const Capture& message) = 0;+};+} // namespace logging+} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/logging/logging.h b/include/onnxruntime/core/common/logging/logging.hnew file mode 100644index 0000000..96c07f0--- /dev/null+++ b/include/onnxruntime/core/common/logging/logging.h@@ -0,0 +1,347 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <atomic>+#include <chrono>+#include <climits>+#include <map>+#include <memory>+#include <mutex>+#include <string>++#include "core/common/common.h"+#include "core/common/logging/capture.h"+#include "core/common/logging/severity.h"++#include "core/common/logging/macros.h"++/*++ Logging overview and expected usage:++ At program startup:+ * Create one or more ISink instances. If multiple, combine using composite_sink.+ * Create a LoggingManager instance with the sink/s with is_default_instance set to true+ * Only one instance should be created in this way, and it should remain valid for+ until the program no longer needs to produce log output.++ You can either use the static default Logger which LoggingManager will create when constructed+ via LoggingManager::DefaultLogger(), or separate Logger instances each with different log ids+ via LoggingManager::CreateLogger.++ The log id is passed to the ISink instance with the sink determining how the log id is used+ in the output.++ LoggingManager+ * creates the Logger instances used by the application+ * provides a static default logger instance+ * owns the log sink instance+ * applies checks on severity and output of user data++ The log macros create a Capture instance to capture the information to log.+ If the severity and/or user filtering settings would prevent logging, no evaluation+ of the log arguments will occur, so no performance cost beyond the severity and user+ filtering check.++ A sink can do further filter as needed.++*/++namespace onnxruntime {+namespace profiling {++enum EventCategory {+ SESSION_EVENT = 0,+ NODE_EVENT,+ EVENT_CATEGORY_MAX+};++/*+Event descriptions for the above session events.+*/+static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = {+ "Session",+ "Node"};++/*+Timing record for all events.+*/+struct EventRecord {+ EventRecord(EventCategory category,+ int process_id,+ int thread_id,+ std::string event_name,+ long long time_stamp,+ long long duration,+ std::unordered_map<std::string, std::string>&& event_args) : cat(category),+ pid(process_id),+ tid(thread_id),+ name(std::move(event_name)),+ ts(time_stamp),+ dur(duration),+ args(event_args) {}+ EventCategory cat;+ int pid;+ int tid;+ std::string name;+ long long ts;+ long long dur;+ std::unordered_map<std::string, std::string> args;+};+} // namespace profiling++namespace logging {++using Timestamp = std::chrono::time_point<std::chrono::system_clock>;++#ifndef NDEBUG+ORT_ATTRIBUTE_UNUSED static bool vlog_enabled = true; // Set directly based on your needs.+#else+constexpr bool vlog_enabled = false; // no VLOG output+#endif++enum class DataType {+ SYSTEM = 0, ///< System data.+ USER = 1 ///< Contains potentially sensitive user data.+};++// Internal log categories.+// Logging interface takes const char* so arbitrary values can also be used.+struct Category {+ static const char* onnxruntime; ///< General output+ static const char* System; ///< Log output regarding interactions with the host system+ // TODO: What other high level categories are meaningful? Model? Optimizer? Execution?+};++class ISink;+class Logger;+class Capture;++/// <summary>+/// The logging manager.+/// Owns the log sink and potentially provides a default Logger instance.+/// Provides filtering based on a minimum LogSeverity level, and of messages with DataType::User if enabled.+/// </summary>+class LoggingManager final {+ public:+ enum InstanceType {+ Default, ///< Default instance of LoggingManager that should exist for the lifetime of the program+ Temporal ///< Temporal instance. CreateLogger(...) should be used, however DefaultLogger() will NOT be provided via this instance.+ };++ /**+ Initializes a new instance of the LoggingManager class.+ @param sink The sink to write to. Use CompositeSink if you need to write to multiple places.+ @param default_min_severity The default minimum severity. Messages with lower severity will be ignored unless+ overridden in CreateLogger.+ @param default_filter_user_data If set to true ignore messages with DataType::USER unless overridden in CreateLogger.+ @param instance_type If InstanceType::Default, this is the default instance of the LoggingManager+ and is expected to exist for the lifetime of the program.+ It creates and owns the default logger that calls to the static DefaultLogger method return.+ @param default_logger_id Logger Id to use for the default logger. nullptr/ignored if instance_type == Temporal.+ @param default_max_vlog_level Default maximum level for VLOG messages to be created unless overridden in CreateLogger.+ Requires a severity of kVERBOSE for VLOG messages to be logged.+ */+ LoggingManager(std::unique_ptr<ISink> sink, Severity default_min_severity, bool default_filter_user_data,+ InstanceType instance_type,+ const std::string* default_logger_id = nullptr,+ int default_max_vlog_level = -1);++ /**+ Creates a new logger instance which will use the provided logger_id and default severity and vlog levels.+ @param logger_id The log identifier.+ @returns A new Logger instance that the caller owns.+ */+ std::unique_ptr<Logger> CreateLogger(const std::string& logger_id);++ /**+ Creates a new logger instance which will use the provided logger_id, severity and vlog levels.+ @param logger_id The log identifier.+ @param min_severity The minimum severity. Requests to create messages with lower severity will be ignored.+ @param filter_user_data If set to true ignore messages with DataType::USER.+ @param max_vlog_level Maximum level for VLOG messages to be created.+ @returns A new Logger instance that the caller owns.+ */+ std::unique_ptr<Logger> CreateLogger(const std::string& logger_id,+ Severity min_severity, bool filter_user_data, int max_vlog_level = -1);++ /**+ Gets the default logger instance if set. Throws if no default logger is currently registered.+ @remarks+ Creating a LoggingManager instance with is_default_instance == true registers a default logger.+ Note that the default logger is only valid until the LoggerManager that registered it is destroyed.+ @returns The default logger if available.+ */+ static const Logger& DefaultLogger();++ /**+ Change the minimum severity level for log messages to be output by the default logger.+ @param severity The severity.+ */+ static void SetDefaultLoggerSeverity(Severity severity);++ /**+ Logs a FATAL level message and creates an exception that can be thrown with error information.+ @param category The log category.+ @param location The location the log message was generated.+ @param format_str The printf format string.+ @param ... The printf arguments.+ @returns A new Logger instance that the caller owns.+ */+ static std::exception LogFatalAndCreateException(const char* category,+ const CodeLocation& location,+ const char* format_str, ...);++ /**+ Logs the message using the provided logger id.+ @param logger_id The log identifier.+ @param message The log message.+ */+ void Log(const std::string& logger_id, const Capture& message) const;++ /**+ Sends a Profiling Event Record to the sink.+ @param Profiling Event Record+ */+ void SendProfileEvent(profiling::EventRecord& eventRecord) const;+ ~LoggingManager();++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(LoggingManager);++ Timestamp GetTimestamp() const noexcept;+ void CreateDefaultLogger(const std::string& logger_id);++ std::unique_ptr<ISink> sink_;+ const Severity default_min_severity_;+ const bool default_filter_user_data_;+ const int default_max_vlog_level_;+ bool owns_default_logger_;++ static Logger* s_default_logger_;++ struct Epochs {+ const std::chrono::time_point<std::chrono::high_resolution_clock> high_res;+ const std::chrono::time_point<std::chrono::system_clock> system;+ const std::chrono::minutes localtime_offset_from_utc;+ };++ static const Epochs& GetEpochs() noexcept;+};++/**+ Logger provides a per-instance log id. Everything else is passed back up to the LoggingManager+*/+class Logger {+ public:+ /**+ Initializes a new instance of the Logger class.+ @param loggingManager The logging manager.+ @param id The identifier for messages coming from this Logger.+ @param severity Minimum severity for messages to be created and logged.+ @param filter_user_data Should USER data be filtered from output.+ @param vlog_level Minimum level for VLOG messages to be created. Note that a severity of kVERBOSE must be provided+ for VLOG messages to be logged.+ */+ Logger(const LoggingManager& loggingManager, std::string id,+ Severity severity, bool filter_user_data, int vlog_level)+ : logging_manager_{&loggingManager},+ id_{id},+ min_severity_{severity},+ filter_user_data_{filter_user_data},+ max_vlog_level_{severity > Severity::kVERBOSE ? -1 : vlog_level} { // disable unless logging VLOG messages+ }++ /**+ Get the minimum severity level for log messages to be output.+ @returns The severity.+ */+ Severity GetSeverity() const noexcept { return min_severity_; }++ /**+ Change the minimum severity level for log messages to be output.+ @param severity The severity.+ */+ void SetSeverity(Severity severity) noexcept { min_severity_ = severity; }++ /**+ Check if output is enabled for the provided LogSeverity and DataType values.+ @param severity The severity.+ @param data_type Type of the data.+ @returns True if a message with these values will be logged.+ */+ bool OutputIsEnabled(Severity severity, DataType data_type) const noexcept {+ return (severity >= min_severity_ && (data_type != DataType::USER || !filter_user_data_));+ }++ /**+ Return the maximum VLOG level allowed.+ */+ int VLOGMaxLevel() const noexcept {+ return max_vlog_level_;+ }++ /**+ Logs the captured message.+ @param message The log message.+ */+ void Log(const Capture& message) const {+ logging_manager_->Log(id_, message);+ }++ /**+ Sends a Profiling Event Record to the sink.+ @param Profiling Event Record+ */+ void SendProfileEvent(profiling::EventRecord& eventRecord) const {+ logging_manager_->SendProfileEvent(eventRecord);+ }++ private:+ const LoggingManager* logging_manager_;+ const std::string id_;+ Severity min_severity_;+ const bool filter_user_data_;+ const int max_vlog_level_;+};++inline const Logger& LoggingManager::DefaultLogger() {+ if (s_default_logger_ == nullptr) {+ // fail early for attempted misuse. don't use logging macros as we have no logger.+ ORT_THROW("Attempt to use DefaultLogger but none has been registered.");+ }++ return *s_default_logger_;+}++inline void LoggingManager::SetDefaultLoggerSeverity(Severity severity) {+ if (s_default_logger_ == nullptr) {+ // fail early for attempted misuse. don't use logging macros as we have no logger.+ ORT_THROW("Attempt to use DefaultLogger but none has been registered.");+ }++ s_default_logger_->SetSeverity(severity);+}++inline Timestamp LoggingManager::GetTimestamp() const noexcept {+ static const Epochs& epochs = GetEpochs();++ const auto high_res_now = std::chrono::high_resolution_clock::now();+ return std::chrono::time_point_cast<std::chrono::system_clock::duration>(+ epochs.system + (high_res_now - epochs.high_res) + epochs.localtime_offset_from_utc);+}++/**+ Return the current thread id.+*/+unsigned int GetThreadId();++/**+ Return the current process id.+*/+unsigned int GetProcessId();++} // namespace logging+} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/logging/macros.h b/include/onnxruntime/core/common/logging/macros.hnew file mode 100644index 0000000..570bc14--- /dev/null+++ b/include/onnxruntime/core/common/logging/macros.h@@ -0,0 +1,209 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+// NOTE: Don't include this file directly. Include logging.h++#define CREATE_MESSAGE(logger, severity, category, datatype) \+ ::onnxruntime::logging::Capture(logger, ::onnxruntime::logging::Severity::k##severity, category, datatype, ORT_WHERE)++/*+ Both printf and stream style logging are supported.+ Not that printf currently has a 2K limit to the message size.++ LOGS_* macros are for stream style+ LOGF_* macros are for printf style++ The Message class captures the log input, and pushes it through the logger in its destructor.++ Use the *FATAL* macros if you want a Severity::kFatal message to also throw.++ There are a few variants to minimize the length of the macro name required in the calling code.+ They are optimized so the shortest names are for the (expected) most common usage. This can be+ tweaked if needed.++ Explicit logger vs LoggingManager::DefaulLogger()+ Default is for a logger instance to be explicitly passed in.+ The logger instance provides an identifier so that log messages from different runs can be separated.++ Variants with DEFAULT in the macro name use the default logger provided by logging manager. This is+ static so accessible from any code, provided a LoggingManager instance created with InstanceType::Default+ exists somewhere. See logging.h for further explanation of the expected setup.++ DataType+ Default uses DataType::SYSTEM.++ Variants with USER in the macro name use DataType::USER. This is data that could be PII, and may need to+ be filtered from output. LoggingManager applies this filtering.++ Category+ Default category is ::onnxruntime::Logging::Category::onnxruntime.++ If you wish to provide a different category, use variants with CATEGORY in the macro name++*/++// Logging with explicit category++// iostream style logging. Capture log info in Message, and push to the logger in ~Message.+#define LOGS_CATEGORY(logger, severity, category) \+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).Stream()++#define LOGS_USER_CATEGORY(logger, severity, category) \+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).Stream()++ // printf style logging. Capture log info in Message, and push to the logger in ~Message.+#define LOGF_CATEGORY(logger, severity, category, format_str, ...) \+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::SYSTEM)) \+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::SYSTEM).CapturePrintf(format_str, ##__VA_ARGS__)++#define LOGF_USER_CATEGORY(logger, severity, category, format_str, ...) \+ if ((logger).OutputIsEnabled(::onnxruntime::logging::Severity::k##severity, ::onnxruntime::logging::DataType::USER)) \+ CREATE_MESSAGE(logger, severity, category, ::onnxruntime::logging::DataType::USER).CapturePrintf(format_str, ##__VA_ARGS__)++ // Logging with category of "onnxruntime"++#define LOGS(logger, severity) \+ LOGS_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGS_USER(logger, severity) \+ LOGS_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime)++ // printf style logging. Capture log info in Message, and push to the logger in ~Message.+#define LOGF(logger, severity, format_str, ...) \+ LOGF_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++#define LOGF_USER(logger, severity, format_str, ...) \+ LOGF_USER_CATEGORY(logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++ /*++ Macros that use the default logger.+ A LoggingManager instance must be currently valid for the default logger to be available.++ */++ // Logging with explicit category++#define LOGS_DEFAULT_CATEGORY(severity, category) \+ LOGS_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category)++#define LOGS_USER_DEFAULT_CATEGORY(severity, category) \+ LOGS_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category)++#define LOGF_DEFAULT_CATEGORY(severity, category, format_str, ...) \+ LOGF_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__)++#define LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ...) \+ LOGF_USER_CATEGORY(::onnxruntime::logging::LoggingManager::DefaultLogger(), severity, category, format_str, ##__VA_ARGS__)++// Logging with category of "onnxruntime"++#define LOGS_DEFAULT(severity) \+ LOGS_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGS_USER_DEFAULT(severity) \+ LOGS_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGF_DEFAULT(severity, format_str, ...) \+ LOGF_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++#define LOGF_USER_DEFAULT(severity, format_str, ...) \+ LOGF_USER_DEFAULT_CATEGORY(severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++ /*++ Conditional logging++ */++ // Logging with explicit category++#define LOGS_CATEGORY_IF(boolean_expression, logger, severity, category) \+ if ((boolean_expression) == true) LOGS_CATEGORY(logger, severity, category)++#define LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \+ if ((boolean_expression) == true) LOGS_DEFAULT_CATEGORY(severity, category)++#define LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, category) \+ if ((boolean_expression) == true) LOGS_USER_CATEGORY(logger, severity, category)++#define LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category) \+ if ((boolean_expression) == true) LOGS_USER_DEFAULT_CATEGORY(severity, category)++#define LOGF_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \+ if ((boolean_expression) == true) LOGF_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__)++#define LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \+ if ((boolean_expression) == true) LOGF_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__)++#define LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, category, format_str, ...) \+ if ((boolean_expression) == true) LOGF_USER_CATEGORY(logger, severity, category, format_str, ##__VA_ARGS__)++#define LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, category, format_str, ...) \+ if ((boolean_expression) == true) LOGF_USER_DEFAULT_CATEGORY(severity, category, format_str, ##__VA_ARGS__)++ // Logging with category of "onnxruntime"++#define LOGS_IF(boolean_expression, logger, severity) \+ LOGS_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGS_DEFAULT_IF(boolean_expression, severity) \+ LOGS_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGS_USER_IF(boolean_expression, logger, severity) \+ LOGS_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGS_USER_DEFAULT_IF(boolean_expression, severity) \+ LOGS_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime)++#define LOGF_IF(boolean_expression, logger, severity, format_str, ...) \+ LOGF_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++#define LOGF_DEFAULT_IF(boolean_expression, severity, format_str, ...) \+ LOGF_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, format_str, ##__VA_ARGS__)++#define LOGF_USER_IF(boolean_expression, logger, severity, format_str, ...) \+ LOGF_USER_CATEGORY_IF(boolean_expression, logger, severity, ::onnxruntime::logging::Category::onnxruntime, \+ format_str, ##__VA_ARGS__)++#define LOGF_USER_DEFAULT_IF(boolean_expression, severity, format_str, ...) \+ LOGF_USER_DEFAULT_CATEGORY_IF(boolean_expression, severity, ::onnxruntime::logging::Category::onnxruntime, \+ format_str, ##__VA_ARGS__)++/*++ Debug verbose logging of caller provided level.+ Disabled in Release builds.+ Use the _USER variants for VLOG statements involving user data that may need to be filtered.+*/+#define VLOGS(logger, level) \+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \+ LOGS_CATEGORY(logger, VERBOSE, "VLOG" #level)++#define VLOGS_USER(logger, level) \+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \+ LOGS_USER_CATEGORY(logger, VERBOSE, "VLOG" #level)++#define VLOGF(logger, level, format_str, ...) \+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \+ LOGF_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__)++#define VLOGF_USER(logger, level, format_str, ...) \+ if (::onnxruntime::logging::vlog_enabled && level <= (logger).VLOGMaxLevel()) \+ LOGF_USER_CATEGORY(logger, VERBOSE, "VLOG" #level, format_str, ##__VA_ARGS__)++ // Default logger variants+#define VLOGS_DEFAULT(level) \+ VLOGS(::onnxruntime::logging::LoggingManager::DefaultLogger(), level)++#define VLOGS_USER_DEFAULT(level) \+ VLOGS_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level)++#define VLOGF_DEFAULT(level, format_str, ...) \+ VLOGF(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__)++#define VLOGF_USER_DEFAULT(level, format_str, ...) \+ VLOGF_USER(::onnxruntime::logging::LoggingManager::DefaultLogger(), level, format_str, ##__VA_ARGS__)diff --git a/include/onnxruntime/core/common/logging/severity.h b/include/onnxruntime/core/common/logging/severity.hnew file mode 100644index 0000000..e43f192--- /dev/null+++ b/include/onnxruntime/core/common/logging/severity.h@@ -0,0 +1,22 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {+namespace logging {+// mild violation of naming convention. the 'k' lets us use token concatenation in the macro+// ::onnxruntime::Logging::Severity::k##severity. It's not legal to have ::onnxruntime::Logging::Severity::##severity+// the uppercase makes the LOG macro usage look as expected for passing an enum value as it will be LOGS(logger, ERROR)+enum class Severity {+ kVERBOSE = 0,+ kINFO = 1,+ kWARNING = 2,+ kERROR = 3,+ kFATAL = 4+};++constexpr const char* SEVERITY_PREFIX = "VIWEF";++} // namespace logging+} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/make_string.h b/include/onnxruntime/core/common/make_string.hnew file mode 100644index 0000000..a6e421a--- /dev/null+++ b/include/onnxruntime/core/common/make_string.h@@ -0,0 +1,82 @@+/**+ * Copyright (c) 2016-present, Facebook, Inc.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+// Portions Copyright (c) Microsoft Corporation++#pragma once++#include <locale>+#include <sstream>++namespace onnxruntime {++namespace detail {+inline void MakeStringImpl(std::ostringstream& /*ss*/) noexcept {+}++template <typename T>+inline void MakeStringImpl(std::ostringstream& ss, const T& t) noexcept {+ ss << t;+}++template <typename T, typename... Args>+inline void MakeStringImpl(std::ostringstream& ss, const T& t, const Args&... args) noexcept {+ MakeStringImpl(ss, t);+ MakeStringImpl(ss, args...);+}+} // namespace detail++/**+ * Makes a string by concatenating string representations of the arguments.+ * This version uses the current locale.+ */+template <typename... Args>+std::string MakeString(const Args&... args) {+ std::ostringstream ss;+ detail::MakeStringImpl(ss, args...);+ return ss.str();+}++/**+ * Makes a string by concatenating string representations of the arguments.+ * This version uses std::locale::classic().+ */+template <typename... Args>+std::string MakeStringWithClassicLocale(const Args&... args) {+ std::ostringstream ss;+ ss.imbue(std::locale::classic());+ detail::MakeStringImpl(ss, args...);+ return ss.str();+}++// MakeString versions for already-a-string types.++inline std::string MakeString(const std::string& str) {+ return str;+}++inline std::string MakeString(const char* cstr) {+ return cstr;+}++inline std::string MakeStringWithClassicLocale(const std::string& str) {+ return str;+}++inline std::string MakeStringWithClassicLocale(const char* cstr) {+ return cstr;+}++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/make_unique.h b/include/onnxruntime/core/common/make_unique.hnew file mode 100644index 0000000..b401f0d--- /dev/null+++ b/include/onnxruntime/core/common/make_unique.h@@ -0,0 +1,148 @@+// Copyright 2017 The Abseil Authors.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// https://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+// -----------------------------------------------------------------------------+// File: make_unique.h+// -----------------------------------------------------------------------------+//+// This header file contains utility functions for managing the creation and+// conversion of smart pointers. This file is an extension to the C+++// standard <memory> library header file.+/* Modifications Copyright (c) Microsoft. */++#pragma once++#include <cstddef>+#include <limits>+#include <memory>+#include <new>+#include <type_traits>+#include <utility>++namespace onnxruntime {++template <typename T>+using remove_extent_t = typename std::remove_extent<T>::type;++namespace memory_internal {++// Traits to select proper overload and return type for `absl::make_unique<>`.+template <typename T>+struct MakeUniqueResult {+ using scalar = std::unique_ptr<T>;+};+template <typename T>+struct MakeUniqueResult<T[]> {+ using array = std::unique_ptr<T[]>;+};+template <typename T, size_t N>+struct MakeUniqueResult<T[N]> {+ using invalid = void;+};++} // namespace memory_internal++// gcc 4.8 has __cplusplus at 201301 but doesn't define make_unique. Other+// supported compilers either just define __cplusplus as 201103 but have+// make_unique (msvc), or have make_unique whenever __cplusplus > 201103 (clang)+#if (__cplusplus > 201103L || defined(_MSC_VER)) && \+ !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 8)+using std::make_unique;+#else+// -----------------------------------------------------------------------------+// Function Template: make_unique<T>()+// -----------------------------------------------------------------------------+//+// Creates a `std::unique_ptr<>`, while avoiding issues creating temporaries+// during the construction process. `absl::make_unique<>` also avoids redundant+// type declarations, by avoiding the need to explicitly use the `new` operator.+//+// This implementation of `absl::make_unique<>` is designed for C++11 code and+// will be replaced in C++14 by the equivalent `std::make_unique<>` abstraction.+// `absl::make_unique<>` is designed to be 100% compatible with+// `std::make_unique<>` so that the eventual migration will involve a simple+// rename operation.+//+// For more background on why `std::unique_ptr<T>(new T(a,b))` is problematic,+// see Herb Sutter's explanation on+// (Exception-Safe Function Calls)[https://herbsutter.com/gotw/_102/].+// (In general, reviewers should treat `new T(a,b)` with scrutiny.)+//+// Example usage:+//+// auto p = make_unique<X>(args...); // 'p' is a std::unique_ptr<X>+// auto pa = make_unique<X[]>(5); // 'pa' is a std::unique_ptr<X[]>+//+// Three overloads of `absl::make_unique` are required:+//+// - For non-array T:+//+// Allocates a T with `new T(std::forward<Args> args...)`,+// forwarding all `args` to T's constructor.+// Returns a `std::unique_ptr<T>` owning that object.+//+// - For an array of unknown bounds T[]:+//+// `absl::make_unique<>` will allocate an array T of type U[] with+// `new U[n]()` and return a `std::unique_ptr<U[]>` owning that array.+//+// Note that 'U[n]()' is different from 'U[n]', and elements will be+// value-initialized. Note as well that `std::unique_ptr` will perform its+// own destruction of the array elements upon leaving scope, even though+// the array [] does not have a default destructor.+//+// NOTE: an array of unknown bounds T[] may still be (and often will be)+// initialized to have a size, and will still use this overload. E.g:+//+// auto my_array = absl::make_unique<int[]>(10);+//+// - For an array of known bounds T[N]:+//+// `absl::make_unique<>` is deleted (like with `std::make_unique<>`) as+// this overload is not useful.+//+// NOTE: an array of known bounds T[N] is not considered a useful+// construction, and may cause undefined behavior in templates. E.g:+//+// auto my_array = absl::make_unique<int[10]>();+//+// In those cases, of course, you can still use the overload above and+// simply initialize it to its desired size:+//+// auto my_array = absl::make_unique<int[]>(10);++// `absl::make_unique` overload for non-array types.+template <typename T, typename... Args>+typename memory_internal::MakeUniqueResult<T>::scalar make_unique(+ Args&&... args) {+ return std::unique_ptr<T>(new T(std::forward<Args>(args)...));+}++// `absl::make_unique` overload for an array T[] of unknown bounds.+// The array allocation needs to use the `new T[size]` form and cannot take+// element constructor arguments. The `std::unique_ptr` will manage destructing+// these array elements.+template <typename T>+typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) {+ return std::unique_ptr<T>(new typename onnxruntime::remove_extent_t<T>[n]());+}++// `absl::make_unique` overload for an array T[N] of known bounds.+// This construction will be rejected.+template <typename T, typename... Args>+typename memory_internal::MakeUniqueResult<T>::invalid make_unique(+ Args&&... /* args */) = delete;+#endif++}\ No newline at end of filediff --git a/include/onnxruntime/core/common/optional.h b/include/onnxruntime/core/common/optional.hnew file mode 100644index 0000000..ab32cf6--- /dev/null+++ b/include/onnxruntime/core/common/optional.h@@ -0,0 +1,24 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <nonstd/optional.hpp>++namespace onnxruntime {++using nonstd::optional;++#ifndef ORT_NO_EXCEPTIONS+using nonstd::bad_optional_access;+#endif++using nonstd::nullopt;+using nonstd::nullopt_t;++using nonstd::in_place;+using nonstd::in_place_t;++using nonstd::make_optional;++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/parse_string.h b/include/onnxruntime/core/common/parse_string.hnew file mode 100644index 0000000..1429859--- /dev/null+++ b/include/onnxruntime/core/common/parse_string.h@@ -0,0 +1,84 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <locale>+#include <sstream>+#include <type_traits>++#include "core/common/common.h"++namespace onnxruntime {++/**+ * Tries to parse a value from an entire string.+ */+template <typename T>+bool TryParseStringWithClassicLocale(const std::string& str, T& value) {+ if (std::is_integral<T>::value && std::is_unsigned<T>::value) {+ // if T is unsigned integral type, reject negative values which will wrap+ if (!str.empty() && str[0] == '-') {+ return false;+ }+ }++ // don't allow leading whitespace+ if (!str.empty() && std::isspace(str[0], std::locale::classic())) {+ return false;+ }++ std::istringstream is{str};+ is.imbue(std::locale::classic());+ T parsed_value{};++ const bool parse_successful =+ is >> parsed_value &&+ is.get() == std::istringstream::traits_type::eof(); // don't allow trailing characters+ if (!parse_successful) {+ return false;+ }++ value = std::move(parsed_value);+ return true;+}++inline bool TryParseStringWithClassicLocale(const std::string& str, std::string& value) {+ value = str;+ return true;+}++inline bool TryParseStringWithClassicLocale(const std::string& str, bool& value) {+ if (str == "0" || str == "False" || str == "false") {+ value = false;+ return true;+ }++ if (str == "1" || str == "True" || str == "true") {+ value = true;+ return true;+ }++ return false;+}++/**+ * Parses a value from an entire string.+ */+template <typename T>+Status ParseStringWithClassicLocale(const std::string& s, T& value) {+ ORT_RETURN_IF_NOT(TryParseStringWithClassicLocale(s, value), "Failed to parse value: \"", value, "\"");+ return Status::OK();+}++/**+ * Parses a value from an entire string.+ */+template <typename T>+T ParseStringWithClassicLocale(const std::string& s) {+ T value{};+ ORT_THROW_IF_ERROR(ParseStringWithClassicLocale(s, value));+ return value;+}++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/spin_pause.h b/include/onnxruntime/core/common/spin_pause.hnew file mode 100644index 0000000..49b71e5--- /dev/null+++ b/include/onnxruntime/core/common/spin_pause.h@@ -0,0 +1,28 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#if defined(_M_AMD64)+#include <intrin.h>+#endif++#if defined(__x86_64__)+#include <xmmintrin.h>+#endif++namespace onnxruntime {++namespace concurrency {++// Intrinsic to use in spin-loops++inline void SpinPause() {+#if defined(_M_AMD64) || defined(__x86_64__)+ _mm_pause();+#endif+}++} // namespace concurrency++} // namespace onnxruntimediff --git a/include/onnxruntime/core/common/status.h b/include/onnxruntime/core/common/status.hnew file mode 100644index 0000000..c107655--- /dev/null+++ b/include/onnxruntime/core/common/status.h@@ -0,0 +1,191 @@+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.+Licensed under the Apache License, Version 2.0 (the "License");+you may not use this file except in compliance with the License.+You may obtain a copy of the License at+ http://www.apache.org/licenses/LICENSE-2.0+Unless required by applicable law or agreed to in writing, software+distributed under the License is distributed on an "AS IS" BASIS,+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+See the License for the specific language governing permissions and+limitations under the License.+==============================================================================*/+// Modifications Copyright (c) Microsoft.++#pragma once++#include <memory>+#include <ostream>+#include <string>+#ifdef _WIN32+#include <winerror.h>+#endif++namespace onnxruntime {+namespace common {++enum StatusCategory {+ NONE = 0,+ SYSTEM = 1,+ ONNXRUNTIME = 2,+};++/**+ Error code for ONNXRuntime.+*/+enum StatusCode {+ OK = 0,+ FAIL = 1,+ INVALID_ARGUMENT = 2,+ NO_SUCHFILE = 3,+ NO_MODEL = 4,+ ENGINE_ERROR = 5,+ RUNTIME_EXCEPTION = 6,+ INVALID_PROTOBUF = 7,+ MODEL_LOADED = 8,+ NOT_IMPLEMENTED = 9,+ INVALID_GRAPH = 10,+ EP_FAIL = 11+};++inline const char* StatusCodeToString(StatusCode status) noexcept {+ switch (status) {+ case StatusCode::OK:+ return "SUCCESS";+ case StatusCode::FAIL:+ return "FAIL";+ case StatusCode::INVALID_ARGUMENT:+ return "INVALID_ARGUMENT";+ case StatusCode::NO_SUCHFILE:+ return "NO_SUCHFILE";+ case StatusCode::NO_MODEL:+ return "NO_MODEL";+ case StatusCode::ENGINE_ERROR:+ return "ENGINE_ERROR";+ case StatusCode::RUNTIME_EXCEPTION:+ return "RUNTIME_EXCEPTION";+ case StatusCode::INVALID_PROTOBUF:+ return "INVALID_PROTOBUF";+ case StatusCode::MODEL_LOADED:+ return "MODEL_LOADED";+ case StatusCode::NOT_IMPLEMENTED:+ return "NOT_IMPLEMENTED";+ case StatusCode::INVALID_GRAPH:+ return "INVALID_GRAPH";+ case StatusCode::EP_FAIL:+ return "EP_FAIL";+ default:+ return "GENERAL ERROR";+ }+}++#ifdef _WIN32+inline HRESULT StatusCodeToHRESULT(StatusCode status) noexcept {+ switch (status)+ {+ case StatusCode::OK:+ return S_OK;+ case StatusCode::FAIL:+ return E_FAIL;+ case StatusCode::INVALID_ARGUMENT:+ return E_INVALIDARG;+ case StatusCode::NO_SUCHFILE:+ return __HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);+ case StatusCode::NO_MODEL:+ return __HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);+ case StatusCode::ENGINE_ERROR:+ return E_FAIL;+ case StatusCode::RUNTIME_EXCEPTION:+ return E_FAIL;+ case StatusCode::INVALID_PROTOBUF:+ return __HRESULT_FROM_WIN32(ERROR_FILE_CORRUPT);+ case StatusCode::MODEL_LOADED:+ return __HRESULT_FROM_WIN32(ERROR_INTERNAL_ERROR);+ case StatusCode::NOT_IMPLEMENTED:+ return E_NOTIMPL;+ case StatusCode::INVALID_GRAPH:+ return __HRESULT_FROM_WIN32(ERROR_FILE_CORRUPT);+ case StatusCode::EP_FAIL:+ return __HRESULT_FROM_WIN32(ERROR_INTERNAL_ERROR);+ default:+ return E_FAIL;+ }+}+#endif++class Status {+ public:+ Status() noexcept = default;++ Status(StatusCategory category, int code, const std::string& msg);++ Status(StatusCategory category, int code, const char* msg);++ Status(StatusCategory category, int code);++ Status(const Status& other)+ : state_((other.state_ == nullptr) ? nullptr : new State(*other.state_)) {}++ Status& operator=(const Status& other) {+ if (state_ != other.state_) {+ if (other.state_ == nullptr) {+ state_.reset();+ } else {+ state_.reset(new State(*other.state_));+ }+ }+ return *this;+ }++ Status(Status&&) = default;+ Status& operator=(Status&&) = default;+ ~Status() = default;++ bool IsOK() const {+ return (state_ == nullptr);+ }++ int Code() const noexcept;++ StatusCategory Category() const noexcept;++ const std::string& ErrorMessage() const noexcept;++ std::string ToString() const;++ bool operator==(const Status& other) const {+ return (this->state_ == other.state_) || (ToString() == other.ToString());+ }++ bool operator!=(const Status& other) const {+ return !(*this == other);+ }++ static Status OK() {+ return Status();+ }++ private:+ static const std::string& EmptyString() noexcept;++ struct State {+ State(StatusCategory cat0, int code0, const std::string& msg0)+ : category(cat0), code(code0), msg(msg0) {}++ State(StatusCategory cat0, int code0, const char* msg0)+ : category(cat0), code(code0), msg(msg0) {}++ const StatusCategory category;+ const int code;+ const std::string msg;+ };++ // As long as Code() is OK, state_ == nullptr.+ std::unique_ptr<State> state_;+};++inline std::ostream& operator<<(std::ostream& out, const Status& status) {+ return out << status.ToString();+}++} // namespace common+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/alloc_kind.h b/include/onnxruntime/core/framework/alloc_kind.hnew file mode 100644index 0000000..4534d08--- /dev/null+++ b/include/onnxruntime/core/framework/alloc_kind.h@@ -0,0 +1,35 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#include <iosfwd>++namespace onnxruntime {+// The ml-Values fall into the following categories with respect to their+// memory management:+// - inference inputs: owned (allocated and freed) by caller, and is by+// default read-only by the runtime.+// - inference outputs: allocated by runtime, ownership transferred to+// caller. TODO: Make sure this semantics is clear in InferenceSession API.+// - weights (constant tensors): can be allocated once (statically), and+// reused by all inference calls within an InferenceSession.+// - tensor values: The lifetimes of these tensor-values are statically+// determined, which is used for memory reuse/sharing optimizations. The+// runtime allocates/frees these values at the right time (as determined+// by the static allocation plan). Note that this is simplified since we+// do not try to optimize for "slice" like ops, where we may be able to+// conditionally reuse memory/data in some cases but not others.+// Generalizing this is future work.++enum class AllocKind {+ kNotSet = -1,+ kAllocate = 0,+ kReuse = 1,+ kPreExisting = 2,+ kAllocateStatically = 3,+ kAllocateOutput = 4,+ kShare = 5+};++std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind);+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.hnew file mode 100644index 0000000..1059c39--- /dev/null+++ b/include/onnxruntime/core/framework/allocator.h@@ -0,0 +1,173 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/common.h"+#include "core/framework/fence.h"+#include "core/session/onnxruntime_c_api.h"+#include "ortdevice.h"+#include "ortmemoryinfo.h"++// This configures the arena based allocator used by ORT+// See docs/C_API.md for details on what these mean and how to choose these values+struct OrtArenaCfg {+ size_t max_mem; // use 0 to allow ORT to choose the default+ int arena_extend_strategy; // use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested+ int initial_chunk_size_bytes; // use -1 to allow ORT to choose the default+ int max_dead_bytes_per_chunk; // use -1 to allow ORT to choose the default+};++namespace onnxruntime {+constexpr const char* CPU = "Cpu";+constexpr const char* CUDA = "Cuda";+constexpr const char* CUDA_PINNED = "CudaPinned";+// TODO: Unify the allocator for CUDA and Tensorrt+constexpr const char* TRT = "Tensorrt";+constexpr const char* TRT_PINNED = "TensorrtPinned";+constexpr const char* MIGRAPHX = "MIGraphX";+constexpr const char* MIGRAPHX_PINNED = "MIGraphXPinned";++constexpr size_t kAllocAlignment = 256;++// forward declaration+class SessionState;++template <typename T>+using IAllocatorUniquePtr = std::unique_ptr<T, std::function<void(T*)>>;++class IAllocator {+ public:+ IAllocator(const OrtMemoryInfo& info) : memory_info_(info) {}+ virtual ~IAllocator() = default;+ /**+ @remarks Use SafeInt when calculating the size of memory to allocate using Alloc.+ */+ virtual void* Alloc(size_t size) = 0;+ virtual void Free(void* p) = 0;+ const OrtMemoryInfo& Info() const { return memory_info_; };++ /**+ optional CreateFence interface, as provider like DML has its own fence+ */+ virtual FencePtr CreateFence(const SessionState* /*unused*/) { return nullptr; }++ static bool CalcMemSizeForArray(size_t nmemb, size_t size, size_t* out) noexcept {+ return CalcMemSizeForArrayWithAlignment(nmemb, size, 0, out);+ }++ /**+ * Calculate the memory size for an array. The size is bounds checked using SafeInt.+ * \tparam alignment must be power of 2+ * \param nmemb Number of members or elements in the array+ * \param size Size of each element+ * \param out Total size required after any alignment is applied+ * \return true, successful. false, overflow+ */+ static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t alignment, size_t* out) noexcept ORT_MUST_USE_RESULT;++ /**+ * https://cwe.mitre.org/data/definitions/190.html+ * \param alignment must be power of 2+ * \param nmemb Number of members or elements in the array+ * \param size Size of each element+ * \param out Total size required after any alignment is applied+ * \return true, successful. false, overflow+ * \remarks This was the original API and was implemented in the header. Replaced with the above version+ * implemented in the .cc file so that the SafeInt dependency is internal.+ */+ template <size_t alignment>+ static bool CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept ORT_MUST_USE_RESULT;++ /**+ * allocate memory for an array which has nmemb items of data, each size bytes long+ */+ void* AllocArray(size_t nmemb, size_t size) {+ size_t len;+ if (!CalcMemSizeForArray(nmemb, size, &len))+ return nullptr;+ return Alloc(len);+ }++ /**+ * allocate memory for an array which has nmemb items of data, each size bytes long+ */+ template <size_t alignment>+ void* AllocArrayWithAlignment(size_t nmemb, size_t size) {+ size_t len;+ if (!CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, &len))+ return nullptr;+ return Alloc(len);+ }++ /**+ Create a std::unique_ptr that is allocated and freed by the provided IAllocator.+ @param allocator The allocator.+ @param count_or_bytes The exact bytes to allocate if T is void, otherwise the number of elements to allocate.+ @returns std::unique_ptr with allocated memory and deleter.+ */+ template <typename T>+ static IAllocatorUniquePtr<T> MakeUniquePtr(std::shared_ptr<IAllocator> allocator, size_t count_or_bytes) {+ if (allocator == nullptr) return nullptr;+ // for now limit to fundamental types. we could support others, but to do so either we or the caller+ // needs to call the dtor for the objects, for buffers allocated on device we don't have destructor+ //static_assert(std::is_fundamental<T>::value, "Fundamental type required as no destructors are called.");++ size_t alloc_size = count_or_bytes;++ // if T is not void, 'count_or_bytes' == number of items so allow for that+ if (!std::is_void<T>::value) {+ // sizeof(void) isn't valid, but the compiler isn't smart enough to ignore that this line isn't+ // reachable if T is void. use std::conditional to 'use' void* in the sizeof call+ if (!CalcMemSizeForArray(count_or_bytes,+ sizeof(typename std::conditional<std::is_void<T>::value, void*, T>::type),+ &alloc_size)) return nullptr;+ }++ return IAllocatorUniquePtr<T>{+ static_cast<T*>(allocator->Alloc(alloc_size)), // allocate+ [=](T* ptr) { // capture 'allocator' by value so it's always valid+ allocator->Free(ptr);+ }};+ }++ private:+ OrtMemoryInfo memory_info_;+};++template <size_t alignment>+bool IAllocator::CalcMemSizeForArrayWithAlignment(size_t nmemb, size_t size, size_t* out) noexcept {+ return CalcMemSizeForArrayWithAlignment(nmemb, size, alignment, out);+}++class CPUAllocator : public IAllocator {+ public:+ explicit CPUAllocator(const OrtMemoryInfo& memory_info) : IAllocator(memory_info) {}++ CPUAllocator() : IAllocator(OrtMemoryInfo(CPU, OrtAllocatorType::OrtDeviceAllocator)) {}++ void* Alloc(size_t size) override;+ void Free(void* p) override;+};++#if defined(USE_MIMALLOC_ARENA_ALLOCATOR)+class MiMallocAllocator : public IAllocator {+ public:+ explicit MiMallocAllocator(const OrtMemoryInfo& memory_info) : IAllocator(memory_info) {}+ MiMallocAllocator() : IAllocator(OrtMemoryInfo(CPU, OrtAllocatorType::OrtDeviceAllocator)) {}++ void* Alloc(size_t size) override;+ void Free(void* p) override;+};++#endif++#if defined(USE_MIMALLOC_ARENA_ALLOCATOR)+using TAllocator = MiMallocAllocator;+#else+using TAllocator = CPUAllocator;+#endif++using AllocatorPtr = std::shared_ptr<IAllocator>;++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/customregistry.h b/include/onnxruntime/core/framework/customregistry.hnew file mode 100644index 0000000..aafe5a5--- /dev/null+++ b/include/onnxruntime/core/framework/customregistry.h@@ -0,0 +1,49 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/status.h"+#include "core/common/logging/logging.h"+#include "core/graph/schema_registry.h"+#include "core/framework/op_kernel.h"+#include "core/framework/kernel_def_builder.h"+#include "core/framework/kernel_registry.h"++namespace onnxruntime {++/**+ Represents a registry that contains both custom kernels and custom schemas.+*/+class CustomRegistry final {+ public:+ CustomRegistry() :+ kernel_registry_(std::make_shared<KernelRegistry>()),+ opschema_registry_(std::make_shared<onnxruntime::OnnxRuntimeOpSchemaRegistry>()) {}++ /**+ * Register a kernel definition together with kernel factory method to this session.+ * If any conflict happened between registered kernel def and built-in kernel def,+ * registered kernel will have higher priority.+ * Call this before invoking Initialize().+ * @return OK if success.+ */+ common::Status RegisterCustomKernel(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator);++ common::Status RegisterCustomKernel(KernelCreateInfo&);++ common::Status RegisterOpSet(std::vector<ONNX_NAMESPACE::OpSchema>& schemas, const std::string& domain,+ int baseline_opset_version, int opset_version);++ const std::shared_ptr<KernelRegistry>& GetKernelRegistry();++ const std::shared_ptr<onnxruntime::OnnxRuntimeOpSchemaRegistry>& GetOpschemaRegistry();++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CustomRegistry);+ std::shared_ptr<KernelRegistry> kernel_registry_;+ std::shared_ptr<onnxruntime::OnnxRuntimeOpSchemaRegistry> opschema_registry_;++};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/data_types.h b/include/onnxruntime/core/framework/data_types.hnew file mode 100644index 0000000..6d4633a--- /dev/null+++ b/include/onnxruntime/core/framework/data_types.h@@ -0,0 +1,1039 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <cstdint>+#include <cstring>+#include <string>+#include <type_traits>+#include <map>+#include <unordered_map>++#include "core/common/common.h"+#include "core/common/exceptions.h"+#include "core/framework/endian.h"+#include "core/graph/onnx_protobuf.h"++struct OrtValue;++namespace ONNX_NAMESPACE {+class TypeProto;+} // namespace ONNX_NAMESPACE++namespace onnxruntime {+/// Predefined registered types++#if !defined(DISABLE_ML_OPS)+//maps (only used by ML ops)+using MapStringToString = std::map<std::string, std::string>;+using MapStringToInt64 = std::map<std::string, int64_t>;+using MapStringToFloat = std::map<std::string, float>;+using MapStringToDouble = std::map<std::string, double>;+using MapInt64ToString = std::map<int64_t, std::string>;+using MapInt64ToInt64 = std::map<int64_t, int64_t>;+using MapInt64ToFloat = std::map<int64_t, float>;+using MapInt64ToDouble = std::map<int64_t, double>;+#endif++//vectors/sequences+#if !defined(DISABLE_ML_OPS)+using VectorMapStringToFloat = std::vector<MapStringToFloat>;+using VectorMapInt64ToFloat = std::vector<MapInt64ToFloat>;+#endif+using VectorString = std::vector<std::string>;+using VectorInt64 = std::vector<int64_t>;++class DataTypeImpl;+class TensorTypeBase;+class SparseTensorTypeBase;+class SequenceTensorTypeBase;+class NonTensorTypeBase;+class PrimitiveDataTypeBase;++// MLFloat16+union MLFloat16 {+ uint16_t val;++ explicit MLFloat16(uint16_t x) : val(x) {}+ MLFloat16() : val(0) {}++ // Taken from https://stackoverflow.com/a/60047308/12627730+ float AsFloat(uint32_t x) const {+ float out = 0.0f;+ std::memcpy(&out, &x, sizeof(x));+ return out;+ }++ // Taken from https://stackoverflow.com/a/60047308/12627730+ uint32_t AsUint(float x) const {+ uint32_t out = 0;+ std::memcpy(&out, &x, sizeof(x));+ return out;+ }++ float HalfToFloat(const uint16_t x) const {+ uint16_t half = x;+ if (endian::native == endian::big) {+ // Taken from https://stackoverflow.com/a/2182184/12627730+ half = (x >> 8) | (x << 8);+ }++ // Taken from https://stackoverflow.com/a/60047308/12627730+ // IEEE-754 16-bit floating-point format (without infinity): 1-5-10, exp-15, +-131008.0, +-6.1035156E-5,+ // +-5.9604645E-8, 3.311 digits+ const uint32_t e = (half & 0x7C00) >> 10; // exponent+ const uint32_t m = (half & 0x03FF) << 13; // mantissa+ // evil log2 bit hack to count leading zeros in denormalized format+ const uint32_t v = AsUint(static_cast<float>(m)) >> 23;+ uint32_t full = (half & 0x8000) << 16 | (e != 0) * ((e + 112) << 23 | m) |+ ((e == 0) & (m != 0)) * ((v - 37) << 23 | ((m << (150 - v)) & 0x007FE000)); // sign : normalized : denormalized++ if (endian::native == endian::big) {+ // Taken from https://stackoverflow.com/a/2182184/12627730+ full = ((full >> 24) & 0xff) | // move byte 3 to byte 0+ ((full << 8) & 0xff0000) | // move byte 1 to byte 2+ ((full >> 8) & 0xff00) | // move byte 2 to byte 1+ ((full << 24) & 0xff000000); // byte 0 to byte 3+ }++ return AsFloat(full);+ }++ operator float() const {+ return HalfToFloat(val);+ }+};++inline bool operator==(const MLFloat16& left, const MLFloat16& right) {+ return left.val == right.val;+}++inline bool operator!=(const MLFloat16& left, const MLFloat16& right) {+ return left.val != right.val;+}++inline bool operator<(const MLFloat16& left, const MLFloat16& right) {+ return left.val < right.val;+}++//BFloat16+struct BFloat16 {+ uint16_t val{0};+ explicit BFloat16() = default;+ explicit BFloat16(uint16_t v) : val(v) {}+ explicit BFloat16(float v) {+ if (endian::native == endian::little) {+ std::memcpy(&val, reinterpret_cast<char*>(&v) + sizeof(uint16_t), sizeof(uint16_t));+ } else {+ std::memcpy(&val, &v, sizeof(uint16_t));+ }+ }++ float ToFloat() const {+ float result;+ char* const first = reinterpret_cast<char*>(&result);+ char* const second = first + sizeof(uint16_t);+ if (endian::native == endian::little) {+ std::memset(first, 0, sizeof(uint16_t));+ std::memcpy(second, &val, sizeof(uint16_t));+ } else {+ std::memcpy(first, &val, sizeof(uint16_t));+ std::memset(second, 0, sizeof(uint16_t));+ }+ return result;+ }++ operator float() const {+ return ToFloat();+ }+};++inline void BFloat16ToFloat(const BFloat16* blf, float* flt, size_t size) {+ auto src = blf;+ auto d = flt;+ for (; size != 0; ++src, ++d, --size) {+ *d = src->ToFloat();+ }+}++inline void FloatToBFloat16(const float* flt, BFloat16* blf, size_t size) {+ auto src = flt;+ auto d = blf;+ for (; size != 0; ++src, ++d, --size) {+ new (d) BFloat16(*src);+ }+}++inline bool operator==(const BFloat16& left, const BFloat16& right) {+ return left.val == right.val;+}++inline bool operator!=(const BFloat16& left, const BFloat16& right) {+ return left.val != right.val;+}++inline bool operator<(const BFloat16& left, const BFloat16& right) {+ return left.val < right.val;+}++// DataTypeImpl pointer as unique DataTypeImpl identifier.+using MLDataType = const DataTypeImpl*;+// be used with class MLValue+using DeleteFunc = void (*)(void*);+using CreateFunc = void* (*)();++/**+ * \brief Base class for MLDataType+ *+ */+class DataTypeImpl {+ public:+ virtual ~DataTypeImpl() = default;++ /**+ * \brief this API will be used to check type compatibility at runtime+ *+ * \param type_proto a TypeProto instance that is constructed for a specific type+ * will be checked against a TypeProto instance contained within a corresponding+ * MLDataType instance.+ */+ virtual bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const = 0;++ virtual size_t Size() const = 0;++ virtual DeleteFunc GetDeleteFunc() const = 0;++ /**+ * \brief Retrieves an instance of TypeProto for+ * a given MLDataType+ * \returns optional TypeProto. Only ONNX types+ has type proto, non-ONNX types will return nullptr.+ */+ virtual const ONNX_NAMESPACE::TypeProto* GetTypeProto() const = 0;++ virtual bool IsTensorType() const {+ return false;+ }++ virtual bool IsTensorSequenceType() const {+ return false;+ }++ virtual bool IsSparseTensorType() const {+ return false;+ }++ // Returns this if this is of tensor-type and null otherwise+ virtual const TensorTypeBase* AsTensorType() const {+ return nullptr;+ }++ virtual const SequenceTensorTypeBase* AsSequenceTensorBase() const {+ return nullptr;+ }++ // Returns this if this is of sparse-tensor-type and null otherwise+ virtual const SparseTensorTypeBase* AsSparseTensorType() const {+ return nullptr;+ }++ virtual const NonTensorTypeBase* AsNonTensorTypeBase() const {+ return nullptr;+ }++ // Returns this if this is one of the primitive data types (specialization of PrimitiveDataTypeBase)+ // and null otherwise+ virtual const PrimitiveDataTypeBase* AsPrimitiveDataType() const {+ return nullptr;+ }++ // Return the type meta that we are using in the runtime.+ template <typename T>+ static MLDataType GetType();++ // Return the types for a concrete tensor type, like Tensor_Float+ template <typename elemT>+ static MLDataType GetTensorType();++ template <typename elemT>+ static MLDataType GetSequenceTensorType();++ // Return the MLDataType for a concrete sparse tensor type.+ template <typename elemT>+ static MLDataType GetSparseTensorType();++ /**+ * Convert an ONNX TypeProto to onnxruntime DataTypeImpl.+ * However, this conversion is lossy. Don't try to use 'this->GetTypeProto()' converting it back.+ * Even though GetTypeProto() will not have the original information, it will still have enough to correctly+ * map to MLDataType.+ * \param proto+ */+ static MLDataType TypeFromProto(const ONNX_NAMESPACE::TypeProto& proto);++ static const TensorTypeBase* TensorTypeFromONNXEnum(int type);+ static const SparseTensorTypeBase* SparseTensorTypeFromONNXEnum(int type);+ static const NonTensorTypeBase* SequenceTensorTypeFromONNXEnum(int type);++ static const char* ToString(MLDataType type);+ static std::vector<std::string> ToString(const std::vector<MLDataType>& types);+ // Registers ONNX_NAMESPACE::DataType (internalized string) with+ // MLDataType. DataType is produced by internalizing an instance of+ // TypeProto contained within MLDataType+ static void RegisterDataType(MLDataType);+ static MLDataType GetDataType(const std::string&);++ static const std::vector<MLDataType>& AllTensorTypes();+ static const std::vector<MLDataType>& AllFixedSizeTensorTypes();+ static const std::vector<MLDataType>& AllSequenceTensorTypes();+ static const std::vector<MLDataType>& AllFixedSizeSequenceTensorTypes();+ static const std::vector<MLDataType>& AllNumericTensorTypes();+ static const std::vector<MLDataType>& AllIEEEFloatTensorTypes();+ static const std::vector<MLDataType>& AllFixedSizeTensorExceptHalfTypes();+ static const std::vector<MLDataType>& AllIEEEFloatTensorExceptHalfTypes();+ static const std::vector<MLDataType>& AllTensorAndSequenceTensorTypes();+ static const std::vector<MLDataType>& AllFixedSizeTensorAndSequenceTensorTypes();+};++std::ostream& operator<<(std::ostream& out, MLDataType data_type);++/*+ * Type registration helpers+ */+namespace data_types_internal {+/// TensorType helpers+///++template <typename T>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType();++template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<float>() {+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT;+}+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint8_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT8;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int8_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT8;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint16_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int16_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int32_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT32;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<int64_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT64;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<std::string>() {+ return ONNX_NAMESPACE::TensorProto_DataType_STRING;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<bool>() {+ return ONNX_NAMESPACE::TensorProto_DataType_BOOL;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<MLFloat16>() {+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<double>() {+ return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint32_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT32;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<uint64_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT64;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorDataType<BFloat16>() {+ return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16;+}++// There is a specialization only for one+// type argument.+template <typename... Types>+struct TensorElementTypeSetter {+ static void SetTensorElementType(ONNX_NAMESPACE::TypeProto&);+ static void SetMapKeyType(ONNX_NAMESPACE::TypeProto&);+ static int32_t GetElementType();+};++/// Is a given type on the list of types?+/// Accepts a list of types and the first argument is the type+/// We are checking if it is listed among those that follow+template <typename T, typename... Types>+struct IsAnyOf;++/// Two types remaining, end of the list+template <typename T, typename Tail>+struct IsAnyOf<T, Tail> : public std::is_same<T, Tail> {+};++template <typename T, typename H, typename... Tail>+struct IsAnyOf<T, H, Tail...> {+ static constexpr bool value = (std::is_same<T, H>::value ||+ IsAnyOf<T, Tail...>::value);+};++/// Tells if the specified type is one of fundamental types+/// that can be contained within a tensor.+/// We do not have raw fundamental types, rather a subset+/// of fundamental types is contained within tensors.+template <typename T>+struct IsTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, uint16_t, int16_t,+ int32_t, int64_t, std::string, bool, MLFloat16,+ double, uint32_t, uint64_t, BFloat16> {+};++/// Use "IsSparseTensorContainedType<T>::value" to test if a type T+/// is permitted as the element-type of a sparse-tensor.++template <typename T>+struct IsSparseTensorContainedType : public IsAnyOf<T, float, uint8_t, int8_t, uint16_t, int16_t,+ int32_t, int64_t, bool, MLFloat16,+ double, uint32_t, uint64_t, BFloat16> {+};++/// This template's Get() returns a corresponding MLDataType+/// It dispatches the call to either GetTensorType<>() or+/// GetType<>()+template <typename T, bool TensorContainedType>+struct GetMLDataType;++template <typename T>+struct GetMLDataType<T, true> {+ static MLDataType Get() {+ return DataTypeImpl::GetTensorType<T>();+ }+};++template <typename T>+struct GetMLDataType<T, false> {+ static MLDataType Get() {+ return DataTypeImpl::GetType<T>();+ }+};++#if !defined(DISABLE_ML_OPS)+/// MapTypes helper API+/// K should always be one of the primitive data types+/// V can be either a primitive type (in which case it is a tensor)+/// or other preregistered types++void CopyMutableMapValue(const ONNX_NAMESPACE::TypeProto&,+ ONNX_NAMESPACE::TypeProto&);++template <typename K, typename V>+struct SetMapTypes {+ static void Set(ONNX_NAMESPACE::TypeProto& proto) {+ TensorElementTypeSetter<K>::SetMapKeyType(proto);+ MLDataType dt = GetMLDataType<V, IsTensorContainedType<V>::value>::Get();+ const auto* value_proto = dt->GetTypeProto();+#ifdef ORT_NO_RTTI+ ORT_ENFORCE(value_proto != nullptr, "expected a registered ONNX type");+#else+ ORT_ENFORCE(value_proto != nullptr, typeid(V).name(),+ " expected to be a registered ONNX type");+#endif+ CopyMutableMapValue(*value_proto, proto);+ }+};+#endif++/// Sequence helpers+///+// Element type is a primitive type so we set it to a tensor<elemT>+void CopyMutableSeqElement(const ONNX_NAMESPACE::TypeProto&,+ ONNX_NAMESPACE::TypeProto&);++template <typename T>+struct SetSequenceType {+ static void Set(ONNX_NAMESPACE::TypeProto& proto) {+ MLDataType dt = GetMLDataType<T, IsTensorContainedType<T>::value>::Get();+ const auto* elem_proto = dt->GetTypeProto();+#ifdef ORT_NO_RTTI+ ORT_ENFORCE(elem_proto != nullptr, "expected a registered ONNX type");+#else+ ORT_ENFORCE(elem_proto != nullptr, typeid(T).name(),+ " expected to be a registered ONNX type");+#endif+ CopyMutableSeqElement(*elem_proto, proto);+ }+};++/// OpaqueTypes helpers+///+void AssignOpaqueDomainName(const char* domain, const char* name,+ ONNX_NAMESPACE::TypeProto& proto);++} // namespace data_types_internal++/// All tensors base+class TensorTypeBase : public DataTypeImpl {+ public:+ static MLDataType Type();++ /// We first compare type_proto pointers and then+ /// if they do not match try to account for the case+ /// where TypeProto was created ad-hoc and not queried from MLDataType+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;++ bool IsTensorType() const override {+ return true;+ }++ const TensorTypeBase* AsTensorType() const override {+ return this;+ }++ size_t Size() const override;++ DeleteFunc GetDeleteFunc() const override;++ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;++ virtual MLDataType GetElementType() const {+ // should never reach here.+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");+ }++ TensorTypeBase(const TensorTypeBase&) = delete;+ TensorTypeBase& operator=(const TensorTypeBase&) = delete;++ protected:+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();++ TensorTypeBase();+ ~TensorTypeBase() override;++ private:+ struct Impl;+ Impl* impl_;+};++/**+ * \brief Tensor type. This type does not have a C++ type associated with+ * it at registration time except the element type. One of the types mentioned+ * above at IsTensorContainedType<> list is acceptable.+ *+ * \details+ * Usage:+ * ORT_REGISTER_TENSOR(ELEMENT_TYPE)+ * Currently all of the Tensors irrespective of the dimensions are mapped to Tensor<type>+ * type. IsCompatible() currently ignores shape.+ */++template <typename elemT>+class TensorType : public TensorTypeBase {+ public:+ static_assert(data_types_internal::IsTensorContainedType<elemT>::value,+ "Requires one of the tensor fundamental types");++ static MLDataType Type();++ /// Tensors only can contain basic data types+ /// that have been previously registered with ONNXRuntime+ MLDataType GetElementType() const override {+ return DataTypeImpl::GetType<elemT>();+ }++ private:+ TensorType() {+ using namespace data_types_internal;+ TensorElementTypeSetter<elemT>::SetTensorElementType(this->mutable_type_proto());+ }+};++/// Common base-class for all sparse-tensors (with different element types).+class SparseTensorTypeBase : public DataTypeImpl {+ public:+ static MLDataType Type();++ bool IsSparseTensorType() const override {+ return true;+ }++ const SparseTensorTypeBase* AsSparseTensorType() const override {+ return this;+ }++ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;++ size_t Size() const override;++ DeleteFunc GetDeleteFunc() const override;++ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;++ virtual MLDataType GetElementType() const {+ // should never reach here.+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");+ }++ SparseTensorTypeBase(const SparseTensorTypeBase&) = delete;+ SparseTensorTypeBase& operator=(const SparseTensorTypeBase&) = delete;++ protected:+ ONNX_NAMESPACE::TypeProto& mutable_type_proto();++ SparseTensorTypeBase();+ ~SparseTensorTypeBase() override;++ private:+ struct Impl;+ Impl* impl_;+};++template <typename elemT>+class SparseTensorType : public SparseTensorTypeBase {+ public:+ static_assert(data_types_internal::IsSparseTensorContainedType<elemT>::value,+ "Requires one of the sparse-tensor fundamental types");++ static MLDataType Type();++ /// Return a MLDataType representing the element-type+ MLDataType GetElementType() const override {+ return DataTypeImpl::GetType<elemT>();+ }++ private:+ SparseTensorType() {+ using namespace data_types_internal;+ TensorElementTypeSetter<elemT>::SetSparseTensorElementType(mutable_type_proto());+ }+};++/**+ * \brief Provide a specialization for your C++ Non-tensor type+ * so your implementation FromDataTypeContainer/ToDataTypeContainer+ * functions correctly. Otherwise you get a default implementation+ * which may not be what you need/want.+ *+ * This class is used to create OrtValue, fetch data from OrtValue via+ * C/C++ APIs+ */+template <class T>+struct NonTensorTypeConverter {+ static void FromContainer(MLDataType /*dtype*/, const void* /*data*/, size_t /*data_size*/, OrtValue& /*output*/) {+ ORT_THROW("Not implemented");+ }+ static void ToContainer(const OrtValue& /*input*/, size_t /*data_size*/, void* /*data*/) {+ ORT_THROW("Not implemented");+ }+};++/**+ * \brief Base type for all non-tensors, maps, sequences and opaques+ */+class NonTensorTypeBase : public DataTypeImpl {+ public:+ size_t Size() const override = 0;++ DeleteFunc GetDeleteFunc() const override = 0;++ virtual CreateFunc GetCreateFunc() const = 0;++ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;++ const NonTensorTypeBase* AsNonTensorTypeBase() const override {+ return this;+ }++ // \brief Override for Non-tensor types to initialize non-tensor CPP+ // data representation from data. The caller of the interface+ // should have a shared definition of the data which is used to initialize+ // CPP data representation. This is used from C API.+ //+ // \param data - pointer to a data container structure non_tensor type specific+ // \param data_size - size of the data container structure, used for rudimentary checks+ // \param output - reference to a default constructed non-tensor type+ // \returns OrtValue+ // \throw if there is an error+ virtual void FromDataContainer(const void* data, size_t data_size, OrtValue& output) const;++ // \brief Override for Non-tensor types to fetch data from the internal CPP data representation+ // The caller of the interface should have a shared definition of the data which is used to initialize+ // CPP data representation. This is used from C API.+ //+ // \param input - OrtValue containing data+ // \param data_size - size of the structure that is being passed for receiving data, used for+ // validation+ // \param data - pointer to receiving data structure+ virtual void ToDataContainer(const OrtValue& input, size_t data_size, void* data) const;++ NonTensorTypeBase(const NonTensorTypeBase&) = delete;+ NonTensorTypeBase& operator=(const NonTensorTypeBase&) = delete;++ protected:+ NonTensorTypeBase();+ ~NonTensorTypeBase() override;++ ONNX_NAMESPACE::TypeProto& mutable_type_proto();++ bool IsMapCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;++ bool IsSequenceCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;++ bool IsOpaqueCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const;++ private:+ struct Impl;+ Impl* impl_;+};++// This is where T is the actual CPPRuntimeType+template <typename T>+class NonTensorType : public NonTensorTypeBase {+ private:+ static void Delete(void* p) {+ delete static_cast<T*>(p);+ }++ public:+ size_t Size() const override {+ return sizeof(T);+ }++ DeleteFunc GetDeleteFunc() const override {+ return &Delete;+ }++ CreateFunc GetCreateFunc() const override {+ return []() -> void* { return new T(); };+ }++ protected:+ NonTensorType() = default;+};++#if !defined(DISABLE_ML_OPS)+/**+ * \brief MapType. Use this type to register+ * mapping types.+ *+ * \param T - cpp type that you wish to register as runtime MapType+ *+ * \details Usage: ORT_REGISTER_MAP(C++Type)+ * The type is required to have mapped_type and+ * key_type defined+ */+template <typename CPPType>+class MapType : public NonTensorType<CPPType> {+ public:+ static_assert(data_types_internal::IsTensorContainedType<typename CPPType::key_type>::value,+ "Requires one of the tensor fundamental types as key");++ static MLDataType Type();++ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {+ return this->IsMapCompatible(type_proto);+ }++ private:+ MapType() {+ using namespace data_types_internal;+ SetMapTypes<typename CPPType::key_type, typename CPPType::mapped_type>::Set(this->mutable_type_proto());+ }+};+#endif++/**+ * \brief SequenceType. Use to register sequence for non-tensor types.+ *+ * \param T - CPP type that you wish to register as Sequence+ * runtime type.+ *+ * \details Usage: ORT_REGISTER_SEQ(C++Type)+ * The type is required to have value_type defined+ */+template <typename CPPType>+class SequenceType : public NonTensorType<CPPType> {+ public:+ static MLDataType Type();++ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {+ return this->IsSequenceCompatible(type_proto);+ }++ private:+ SequenceType() {+ data_types_internal::SetSequenceType<typename CPPType::value_type>::Set(this->mutable_type_proto());+ }+};++/**+ * \brief SequenceTensorTypeBase serves as a base type class for+ * Tensor sequences. Akin TensorTypeBase.+ * Runtime representation is always TensorSeq.+ */+class SequenceTensorTypeBase : public DataTypeImpl {+ public:+ static MLDataType Type();++ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override;++ bool IsTensorSequenceType() const override {+ return true;+ }++ const SequenceTensorTypeBase* AsSequenceTensorBase() const override {+ return this;+ }++ virtual MLDataType GetElementType() const {+ // should never reach here.+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");+ }++ size_t Size() const override;++ DeleteFunc GetDeleteFunc() const override;++ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const override;++ SequenceTensorTypeBase(const SequenceTensorTypeBase&) = delete;+ SequenceTensorTypeBase& operator=(const SequenceTensorTypeBase&) = delete;++ protected:+ SequenceTensorTypeBase();+ ~SequenceTensorTypeBase();++ ONNX_NAMESPACE::TypeProto& mutable_type_proto();++ private:+ struct Impl;+ Impl* impl_;+};++/**+ * \brief SequenceTensorType. Use to register sequence for non-tensor types.+ *+ * \param CPPRuntime - We always use TensorSeq+ *+ * \param TensorElemType - one of the primitive types+ *+ * \details Usage: ORT_REGISTER_SEQ_TENSOR_TYPE()+ * The type is required to have value_type defined+ */+template <typename TensorElemType>+class SequenceTensorType : public SequenceTensorTypeBase {+ public:+ static MLDataType Type();++ /// Return a MLDataType representing the element-type+ MLDataType GetElementType() const override {+ return DataTypeImpl::GetType<TensorElemType>();+ }++ private:+ SequenceTensorType() {+ data_types_internal::SetSequenceType<TensorElemType>::Set(this->mutable_type_proto());+ }+};++/**+ * \brief OpaqueType+ *+ * \param T - cpp runtume that implements the Opaque type+ *+ * \param const char D[] - domain must be extern to be unique+ *+ * \param const char N[] - name must be extern to be unique+ *+ * \details Only one CPP type can be associated with a particular+ * OpaqueType registration+ *+ */+template <typename T, const char D[], const char N[]>+class OpaqueType : public NonTensorType<T> {+ public:+ static MLDataType Type();++ bool IsCompatible(const ONNX_NAMESPACE::TypeProto& type_proto) const override {+ return this->IsOpaqueCompatible(type_proto);+ }++ void FromDataContainer(const void* data, size_t data_size, OrtValue& output) const override {+ NonTensorTypeConverter<T>::FromContainer(this, data, data_size, output);+ }++ void ToDataContainer(const OrtValue& input, size_t data_size, void* data) const override {+ NonTensorTypeConverter<T>::ToContainer(input, data_size, data);+ }++ private:+ OpaqueType() {+ data_types_internal::AssignOpaqueDomainName(D, N, this->mutable_type_proto());+ }+};++/**+ * \brief PrimitiveDataTypeBase+ * Base class for primitive Tensor contained types+ *+ * \details This class contains an integer constant that can be+ * used for input data type dispatching+ *+ */+class PrimitiveDataTypeBase : public DataTypeImpl {+ public:+ bool IsCompatible(const ONNX_NAMESPACE::TypeProto&) const override {+ return false;+ }++ const PrimitiveDataTypeBase* AsPrimitiveDataType() const override final {+ return this;+ }++ const ONNX_NAMESPACE::TypeProto* GetTypeProto() const final {+ return nullptr;+ }++ int32_t GetDataType() const {+ return data_type_;+ }++ protected:+ PrimitiveDataTypeBase() = default;++ void SetDataType(int32_t data_type) {+ data_type_ = data_type;+ }++ private:+ int32_t data_type_;+};++/**+ * \brief PrimitiveDataType+ * Typed specialization for primitive types.+ * Concrete instances of this class are used by Tensor.+ *+ * \param T - primitive data type+ *+ */+template <typename T>+class PrimitiveDataType : public PrimitiveDataTypeBase {+ private:+ static void Delete(void* p) {+ delete static_cast<T*>(p);+ }++ public:+ static MLDataType Type();++ size_t Size() const override {+ return sizeof(T);+ }++ DeleteFunc GetDeleteFunc() const override {+ return &Delete;+ }++ private:+ PrimitiveDataType() {+ this->SetDataType(data_types_internal::TensorElementTypeSetter<T>::GetElementType());+ }+};++// Explicit specialization of base class template function+// is only possible within the enclosing namespace scope,+// thus a simple way to pre-instantiate a given template+// at a registration time does not currently work and the macro+// is needed.+#define ORT_REGISTER_TENSOR_TYPE(ELEM_TYPE) \+ template <> \+ MLDataType TensorType<ELEM_TYPE>::Type() { \+ static TensorType<ELEM_TYPE> tensor_type; \+ return &tensor_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetTensorType<ELEM_TYPE>() { \+ return TensorType<ELEM_TYPE>::Type(); \+ }++#define ORT_REGISTER_SPARSE_TENSOR_TYPE(ELEM_TYPE) \+ template <> \+ MLDataType SparseTensorType<ELEM_TYPE>::Type() { \+ static SparseTensorType<ELEM_TYPE> tensor_type; \+ return &tensor_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetSparseTensorType<ELEM_TYPE>() { \+ return SparseTensorType<ELEM_TYPE>::Type(); \+ }++#if !defined(DISABLE_ML_OPS)+#define ORT_REGISTER_MAP(TYPE) \+ template <> \+ MLDataType MapType<TYPE>::Type() { \+ static MapType<TYPE> map_type; \+ return &map_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetType<TYPE>() { \+ return MapType<TYPE>::Type(); \+ }+#endif++#define ORT_REGISTER_SEQ(TYPE) \+ template <> \+ MLDataType SequenceType<TYPE>::Type() { \+ static SequenceType<TYPE> sequence_type; \+ return &sequence_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetType<TYPE>() { \+ return SequenceType<TYPE>::Type(); \+ }++#define ORT_REGISTER_SEQ_TENSOR_TYPE(ELEM_TYPE) \+ template <> \+ MLDataType SequenceTensorType<ELEM_TYPE>::Type() { \+ static SequenceTensorType<ELEM_TYPE> sequence_tensor_type; \+ return &sequence_tensor_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetSequenceTensorType<ELEM_TYPE>() { \+ return SequenceTensorType<ELEM_TYPE>::Type(); \+ }++#define ORT_REGISTER_PRIM_TYPE(TYPE) \+ template <> \+ MLDataType PrimitiveDataType<TYPE>::Type() { \+ static PrimitiveDataType<TYPE> prim_data_type; \+ return &prim_data_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetType<TYPE>() { \+ return PrimitiveDataType<TYPE>::Type(); \+ }++#define ORT_REGISTER_OPAQUE_TYPE(CPPType, Domain, Name) \+ template <> \+ MLDataType OpaqueType<CPPType, Domain, Name>::Type() { \+ static OpaqueType<CPPType, Domain, Name> opaque_type; \+ return &opaque_type; \+ } \+ template <> \+ MLDataType DataTypeImpl::GetType<CPPType>() { \+ return OpaqueType<CPPType, Domain, Name>::Type(); \+ }+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/data_types_internal.h b/include/onnxruntime/core/framework/data_types_internal.hnew file mode 100644index 0000000..1f154fc--- /dev/null+++ b/include/onnxruntime/core/framework/data_types_internal.h@@ -0,0 +1,520 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <assert.h>+#include <stdint.h>+#include <string>+#include <vector>++#include "core/common/common.h"+#include "core/framework/data_types.h"+#include "core/graph/onnx_protobuf.h"++#ifdef _MSC_VER+#pragma warning(push)+//TODO: fix the warning in CallableDispatchableRetHelper+#pragma warning(disable : 4702)+#endif+namespace onnxruntime {+namespace utils {++template <typename T>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType() {+ return ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED;+}++template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<float>() {+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT;+}+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint8_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT8;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int8_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT8;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint16_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int16_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int32_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT32;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<int64_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_INT64;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<std::string>() {+ return ONNX_NAMESPACE::TensorProto_DataType_STRING;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<bool>() {+ return ONNX_NAMESPACE::TensorProto_DataType_BOOL;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<MLFloat16>() {+ return ONNX_NAMESPACE::TensorProto_DataType_FLOAT16;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<double>() {+ return ONNX_NAMESPACE::TensorProto_DataType_DOUBLE;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint32_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT32;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<uint64_t>() {+ return ONNX_NAMESPACE::TensorProto_DataType_UINT64;+};+template <>+constexpr ONNX_NAMESPACE::TensorProto_DataType ToTensorProtoElementType<BFloat16>() {+ return ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16;+};++ // The following primitives are strongly recommended for switching on tensor input datatypes for+ // kernel implementations.+ //+ // 1) If you need to handle all of the primitive tensor contained datatypes, the best choice would be macros+ // DispatchOnTensorType or DispatchOnTensorTypeWithReturn. Use inline wrappers so your function can be invoked as function<T>().+ // 2) if you have a few types, use Tensor.IsDataType<T>()/IsDataTypeString() or use utils::IsPrimitiveDataType<T>()+ // if you have a standalone MLDatatType with a sequence of if/else statements.+ // 3) For something in between, we suggest to use CallDispatcher pattern.+ //+ // Invoking DataTypeImpl::GetType<T>() for switching on input types is discouraged and should be avoided.+ // Every primitive type carries with it an integer constant that can be used for quick switching on types.++#define DispatchOnTensorType(tensor_type, function, ...) \+ switch (tensor_type->AsPrimitiveDataType()->GetDataType()) { \+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: \+ function<float>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_BOOL: \+ function<bool>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: \+ function<double>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_STRING: \+ function<std::string>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT8: \+ function<int8_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT8: \+ function<uint8_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT16: \+ function<int16_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT16: \+ function<uint16_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT32: \+ function<int32_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT32: \+ function<uint32_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT64: \+ function<int64_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT64: \+ function<uint64_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: \+ function<MLFloat16>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: \+ function<BFloat16>(__VA_ARGS__); \+ break; \+ default: \+ ORT_ENFORCE(false, "Unknown tensor type of ", tensor_type); \+ }++#define DispatchOnTensorTypeWithReturn(tensor_type, retval, function, ...) \+ switch (tensor_type->AsPrimitiveDataType()->GetDataType()) { \+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: \+ retval = function<float>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_BOOL: \+ retval = function<bool>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_DOUBLE: \+ retval = function<double>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_STRING: \+ retval = function<std::string>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT8: \+ retval = function<int8_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT8: \+ retval = function<uint8_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT16: \+ retval = function<uint16_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT16: \+ retval = function<int16_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT32: \+ retval = function<int32_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT32: \+ retval = function<uint32_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_INT64: \+ retval = function<int64_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_UINT64: \+ retval = function<uint64_t>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: \+ retval = function<MLFloat16>(__VA_ARGS__); \+ break; \+ case ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16: \+ retval = function<BFloat16>(__VA_ARGS__); \+ break; \+ default: \+ ORT_ENFORCE(false, "Unknown tensor type of ", tensor_type); \+ }++////////////////////////////////////////////////////////////////////////////////+/// Use the following primitives if you have a few types to switch on so you+// can write a short sequence of if/else statements.++// This is a frequently used check so we make a separate utility function.+inline bool IsDataTypeString(MLDataType dt_type) {+ auto prim_type = dt_type->AsPrimitiveDataType();+ return (prim_type != nullptr && prim_type->GetDataType() == ONNX_NAMESPACE::TensorProto_DataType_STRING);+}++// Test if MLDataType is a concrete type of PrimitiveDataTypeBase+// and it is T+template <class T>+inline bool IsPrimitiveDataType(MLDataType dt_type) {+ auto prim_type = dt_type->AsPrimitiveDataType();+ return (prim_type != nullptr && prim_type->GetDataType() == ToTensorProtoElementType<T>());+}++// Use after AsPrimitiveDataType() is successful+// Check if PrimitiveDataTypeBase is of type T+template <class T>+inline bool IsPrimitiveDataType(const PrimitiveDataTypeBase* prim_type) {+ assert(prim_type != nullptr);+ return prim_type->GetDataType() == ToTensorProtoElementType<T>();+}++// This implementation contains a workaround for GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47226+// GCC until very recently does not support template parameter pack expansion within lambda context.+namespace mltype_dispatcher_internal {+// T - type handled by this helper+struct CallableDispatchableHelper {+ int32_t dt_type_; // Type currently dispatched+ size_t called_;++ explicit CallableDispatchableHelper(int32_t dt_type) noexcept : dt_type_(dt_type), called_(0) {}++ // Must return integer to be in a expandable context+ template <class T, class Fn, class... Args>+ int Invoke(Fn&& fn, Args&&... args) {+ if (utils::ToTensorProtoElementType<T>() == dt_type_) {+ std::forward<Fn>(fn)(std::forward<Args>(args)...);+ ++called_;+ }+ return 0;+ }+};++// Default policy is to throw with no return type.+template <class Ret>+struct UnsupportedTypeDefaultPolicy {+ Ret operator()(int32_t dt_type) const {+ ORT_THROW("Unsupported data type: ", dt_type);+ }+};++// Helper with the result type+template <class Ret, class UnsupportedPolicy = UnsupportedTypeDefaultPolicy<Ret>>+struct CallableDispatchableRetHelper {+ int32_t dt_type_; // Type currently dispatched+ size_t called_;+ Ret result_;++ explicit CallableDispatchableRetHelper(int32_t dt_type) noexcept : dt_type_(dt_type), called_(0), result_() {}++ Ret Get() {+ // See if there were multiple invocations.It is a bug.+ ORT_ENFORCE(called_ < 2, "Check for duplicate types in MLTypeCallDispatcherRet");+ // No type was invoked+ if (called_ == 0) {+ result_ = UnsupportedPolicy()(dt_type_);+ }+ return result_;+ }++ // Must return integer to be in a expandable context+ template <class T, class Fn, class... Args>+ int Invoke(Fn&& fn, Args&&... args) {+ if (utils::ToTensorProtoElementType<T>() == dt_type_) {+ result_ = std::forward<Fn>(fn)(std::forward<Args>(args)...);+ ++called_;+ }+ return 0;+ }+};++} // namespace mltype_dispatcher_internal++// This class helps to efficiently dispatch calls for templated+// kernel implementation functions that has no return value.+// If your implementation function must return a value such as Status+// Use MLTypeCallDispatcherRet class.+//+// The first template parameter is a template<T> struct/class functor+// that must implement operator() with arbitrary number of arguments+// and void return turn. It must return Ret type if you are using MLTypeCallDispatcherRet.+// Fn must be default constructible.+//+// Types is a type list that are supported by this kernel implementation.+// There should be no duplicate types. An exception will be thrown if there+// a duplicate.+//+// The constructor accepts an enum that is obtained from+// input_tensor->DataType()->AsPrimitiveType()->GetDataType().+// Fn will be called only once the type designated by dt_type value.+// If current dt_type is not handled, the Dispatcher will throw an exception.+//+template <template <typename> class Fn, typename... Types>+class MLTypeCallDispatcher {+ int32_t dt_type_;++ public:+ explicit MLTypeCallDispatcher(int32_t dt_type) noexcept : dt_type_(dt_type) {}++ template <typename... Args>+ void Invoke(Args&&... args) const {+ mltype_dispatcher_internal::CallableDispatchableHelper helper(dt_type_);+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};+ ORT_UNUSED_PARAMETER(results);+ ORT_ENFORCE(helper.called_ < 2, "Check for duplicate types in MLTypeCallDispatcher");+ ORT_ENFORCE(helper.called_ == 1, "Unsupported data type: ", dt_type_);+ }+};++// Version of the MLTypeDispatcher with a return type.+// Return type of Fn must return type convertible to Ret+// The value of the return type will be the return value+// of the function for type T which was specified for execution.+template <class Ret, template <typename> class Fn, typename... Types>+class MLTypeCallDispatcherRet {+ int32_t dt_type_;++ public:+ explicit MLTypeCallDispatcherRet(int32_t dt_type) noexcept : dt_type_(dt_type) {}++ template <typename... Args>+ Ret Invoke(Args&&... args) const {+ mltype_dispatcher_internal::CallableDispatchableRetHelper<Ret> helper(dt_type_);+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};+ ORT_UNUSED_PARAMETER(results);+ return helper.Get();+ }++ template <class UnsupportedPolicy, typename... Args>+ Ret InvokeWithUnsupportedPolicy(Args&&... args) const {+ mltype_dispatcher_internal::CallableDispatchableRetHelper<Ret, UnsupportedPolicy> helper(dt_type_);+ int results[] = {0, helper.template Invoke<Types>(Fn<Types>(), std::forward<Args>(args)...)...};+ ORT_UNUSED_PARAMETER(results);+ return helper.Get();+ }+};++// Version of the MLTypeDispatcher that has an input type which is passed through ('carried')+// as the first type parameter in the call to Fn when dispatching.+template <typename TCarried, template <typename, typename> class Fn, typename... Types>+class MLTypeCallDispatcherWithCarriedType {+ int32_t dt_type_;++ public:+ explicit MLTypeCallDispatcherWithCarriedType(int32_t dt_type) noexcept : dt_type_(dt_type) {}++ template <typename... Args>+ void Invoke(Args&&... args) const {+ mltype_dispatcher_internal::CallableDispatchableHelper helper(dt_type_);+ int results[] = {0, helper.template Invoke<Types>(Fn<TCarried, Types>(), std::forward<Args>(args)...)...};+ ORT_UNUSED_PARAMETER(results);+ ORT_ENFORCE(helper.called_ < 2, "Check for duplicate types in MLTypeCallDispatcher");+ ORT_ENFORCE(helper.called_ == 1, "Unsupported data type: ", dt_type_);+ }+};++namespace data_types_internal {++enum class ContainerType : uint16_t {+ kUndefined = 0,+ kTensor = 1,+ kMap = 2,+ kSequence = 3,+ kOpaque = 4+};++class TypeNode {+ // type_ is a TypeProto value case enum+ // that may be a kTypeTensor, kTypeMap, kTypeSequence+ // prim_type_ is a TypeProto_DataType enum that has meaning+ // - for Tensor then prim_type_ is the contained type+ // - for Map prim_type is the key type. Next entry describes map value+ // - For sequence prim_type_ is not used and has no meaning. Next entry+ // describes the value for the sequence+ // Tensor is always the last entry as it describes a contained primitive type.+ ContainerType type_;+ uint16_t prim_type_;++ public:+ TypeNode(ContainerType type, int32_t prim_type) noexcept {+ type_ = type;+ prim_type_ = static_cast<uint16_t>(prim_type);+ }++ bool IsType(ContainerType type) const noexcept {+ return type_ == type;+ }++ bool IsPrimType(int32_t prim_type) const noexcept {+ return prim_type_ == static_cast<uint16_t>(prim_type);+ }+};++} // namespace data_types_internal++////////////////////////////////////////////////////////////////////+/// Provides generic interface to test whether MLDataType is a Sequence,+/// Map or an Opaque type including arbitrary recursive definitions+/// without querying DataTypeImpl::GetType<T> for all known complex types++// T is a sequence contained element type+// If returns true then we know that the runtime+// representation is std::vector<T>+// T itself can be a runtime representation of another+// sequence, map, opaque type or a tensor+//+// That is it can be std::vector or a std::map+// If T is a primitive type sequence is tested whether it contains+// tensors of that type+//+// If T is an opaque type, then it is only tested to be opaque but not exactly+// a specific opaque type. To Test for a specific Opaque type use IsOpaqueType() below+//+// This class examines the supplied MLDataType and records+// its information in a vector so any subsequent checks for Sequences and Maps+// are quick.+class ContainerChecker {+ using Cont = std::vector<data_types_internal::TypeNode>;+ Cont types_;++ // Default IsContainerOfType is for Opaque type+ template <class T>+ struct IsContainerOfType {+ static bool check(const Cont& c, size_t index) {+ if (index >= c.size()) {+ return false;+ }+ return c[index].IsType(data_types_internal::ContainerType::kOpaque);+ }+ };++ // Handles the case where sequence element is also a sequence+ template <class T>+ struct IsContainerOfType<std::vector<T>> {+ static bool check(const Cont& c, size_t index) {+ if (index >= c.size()) {+ return false;+ }+ if (c[index].IsType(data_types_internal::ContainerType::kSequence)) {+ ORT_ENFORCE(++index < c.size(), "Sequence is missing type entry for its element");+ constexpr int32_t prim_type = ToTensorProtoElementType<T>();+ // Check if this is a primitive type and it matches+ if (prim_type != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED) {+ return c[index].IsType(data_types_internal::ContainerType::kTensor) &&+ c[index].IsPrimType(prim_type);+ } else {+ // T is not primitive, check next entry for non-primitive proto+ return IsContainerOfType<T>::check(c, index);+ }+ }+ return false;+ }+ };++ template <class K, class V>+ struct IsContainerOfType<std::map<K, V>> {+ static bool check(const Cont& c, size_t index) {+ static_assert(ToTensorProtoElementType<K>() != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED,+ "Map Key can not be a non-primitive type");+ if (index >= c.size()) {+ return false;+ }+ if (!c[index].IsType(data_types_internal::ContainerType::kMap)) {+ return false;+ }+ constexpr int32_t key_type = ToTensorProtoElementType<K>();+ if (!c[index].IsPrimType(key_type)) {+ return false;+ }+ ORT_ENFORCE(++index < c.size(), "Map is missing type entry for its value");+ constexpr int32_t val_type = ToTensorProtoElementType<V>();+ if (val_type != ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED) {+ return c[index].IsType(data_types_internal::ContainerType::kTensor) &&+ c[index].IsPrimType(val_type);+ }+ return IsContainerOfType<V>::check(c, index);+ }+ };++ public:+ explicit ContainerChecker(MLDataType);+ ~ContainerChecker() = default;++ bool IsMap() const noexcept {+ assert(!types_.empty());+ return types_[0].IsType(data_types_internal::ContainerType::kMap);+ }++ bool IsSequence() const noexcept {+ assert(!types_.empty());+ return types_[0].IsType(data_types_internal::ContainerType::kSequence);+ }++ template <class T>+ bool IsSequenceOf() const {+ assert(!types_.empty());+ return IsContainerOfType<std::vector<T>>::check(types_, 0);+ }++ template <class K, class V>+ bool IsMapOf() const {+ assert(!types_.empty());+ return IsContainerOfType<std::map<K, V>>::check(types_, 0);+ }+};++bool IsOpaqueType(MLDataType ml_type, const char* domain, const char* name);++} // namespace utils+} // namespace onnxruntime++#ifdef _MSC_VER+#pragma warning(pop)+#endifdiff --git a/include/onnxruntime/core/framework/endian.h b/include/onnxruntime/core/framework/endian.hnew file mode 100644index 0000000..629fb78--- /dev/null+++ b/include/onnxruntime/core/framework/endian.h@@ -0,0 +1,27 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {++// the semantics of this enum should match std::endian from C++20+enum class endian {+#if defined(_WIN32)+ little = 0,+ big = 1,+ native = little,+#elif defined(__GNUC__) || defined(__clang__)+ little = __ORDER_LITTLE_ENDIAN__,+ big = __ORDER_BIG_ENDIAN__,+ native = __BYTE_ORDER__,+#else+#error onnxruntime::endian is not implemented in this environment.+#endif+};++static_assert(+ endian::native == endian::little || endian::native == endian::big,+ "Only little-endian or big-endian native byte orders are supported.");++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/execution_provider.h b/include/onnxruntime/core/framework/execution_provider.hnew file mode 100644index 0000000..3c877f2--- /dev/null+++ b/include/onnxruntime/core/framework/execution_provider.h@@ -0,0 +1,278 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#ifndef PROVIDER_BRIDGE_PROVIDER+#include <map>+#include <unordered_map>+#include <unordered_set>++#include "core/common/status.h"+#include "core/common/logging/logging.h"+#include "core/framework/tensor.h"+#include "core/framework/data_transfer.h"++namespace onnxruntime {++class GraphViewer;+class Node;+struct ComputeCapability;+class KernelRegistry;+class KernelRegistryManager;+} // namespace onnxruntime+#endif++#include "core/framework/provider_options.h"+#include "core/framework/func_api.h"++namespace onnxruntime {++/**+ Logical device representation.+*/+using AllocatorMap = std::map<int, AllocatorPtr>;+using MemoryInfoSet = std::set<OrtMemoryInfo>;++// if we are export the fused function to dll, the function will still in the same binary as onnxruntime+// use std function to give execution provider some chance to capture some state.+using CreateFunctionStateFunc = std::function<int(ComputeContext*, FunctionState*)>;+using ComputeFunc = std::function<Status(FunctionState, const OrtApi*, OrtKernelContext*)>;+using DestroyFunctionStateFunc = std::function<void(FunctionState)>;++struct NodeComputeInfo {+ CreateFunctionStateFunc create_state_func;+ ComputeFunc compute_func;+ DestroyFunctionStateFunc release_state_func;+};++class IExecutionProvider {+ protected:+ IExecutionProvider(const std::string& type, bool use_metadef_id_creator = false)+ : type_{type} {+ if (use_metadef_id_creator) {+ metadef_id_generator_ = onnxruntime::make_unique<ModelMetadefIdGenerator>();+ }+ }++ public:+ virtual ~IExecutionProvider() = default;++ /**+ Get all IAllocators for <*this> execution provider.+ */+ const std::vector<AllocatorPtr>& GetAllocators() const {+ return allocator_list_;+ }++ /**+ * Get an allocator with specified device id and MemType. Return nullptr if it doesn't exist+ */+ virtual AllocatorPtr GetAllocator(int id, OrtMemType mem_type) const;++ /**+ * Returns a data transfer object that implements methods to copy to and+ * from this device.+ * If no copy is required for the successful operation of this provider,+ * return a nullptr.+ */+ virtual std::unique_ptr<onnxruntime::IDataTransfer> GetDataTransfer() const {+ return nullptr;+ }++ /**+ Get execution provider's capability for the specified <graph>.+ Return a bunch of IndexedSubGraphs <*this> execution provider can run if+ the sub-graph contains only one node or can fuse to run if the sub-graph+ contains more than one node. The node indexes contained in sub-graphs may+ have overlap, and it's ONNXRuntime's responsibility to do the partition+ and decide whether a node will be assigned to <*this> execution provider.+ */+ virtual std::vector<std::unique_ptr<ComputeCapability>>+ GetCapability(const onnxruntime::GraphViewer& graph_viewer,+ const std::vector<const KernelRegistry*>& kernel_registries) const;++ /**+ Get kernel registry per execution provider type.+ The KernelRegistry share pointer returned is shared across sessions.++ NOTE: this is a tricky but final solution to achieve following goals,+ 1. The execution provider type based kernel registry should be shared+ across sessions.+ Only one copy of this kind of kernel registry exists in ONNXRuntime+ with multiple sessions/models.+ 2. Adding an execution provider into ONNXRuntime does not need to touch ONNXRuntime+ frameowrk/session code.+ 3. onnxruntime (framework/session) does not depend on any specific+ execution provider lib.+ */+ virtual std::shared_ptr<KernelRegistry> GetKernelRegistry() const { return nullptr; }++ /**+ Get the device id of current execution provider+ */+ virtual int GetDeviceId() const { return -1; };++ /**+ Get execution provider's configuration options.+ */+ virtual ProviderOptions GetProviderOptions() const { return {}; }++ /**+ Returns an opaque handle whose exact type varies based on the provider+ and is interpreted accordingly by the corresponding kernel implementation.+ For Direct3D operator kernels, this may return an IUnknown supporting+ QueryInterface to ID3D12GraphicsCommandList1.+ */+ virtual const void* GetExecutionHandle() const noexcept {+ return nullptr;+ }++ /**+ @return type of the execution provider; should match that set in the node+ through the SetExecutionProvider API. Example valid return values are:+ kCpuExecutionProvider, kCudaExecutionProvider+ */+ const std::string& Type() const { return type_; }++ /**+ Blocks until the device has completed all preceding requested tasks.+ Currently this is primarily used by the IOBinding object to ensure that all+ inputs have been copied to the device before execution begins.+ */+ virtual common::Status Sync() const { return Status::OK(); }++ /**+ Called when InferenceSession::Run started+ NOTE that due to async execution in provider, the actual work of previous+ Run may not be finished on device This function should be regarded as the+ point after which a new Run would start to submit commands from CPU+ */+ virtual common::Status OnRunStart() { return Status::OK(); }++ /**+ Called when InferenceSession::Run ended+ NOTE that due to async execution in provider, the actual work of this Run+ may not be finished on device This function should be regarded as the point+ that all commands of current Run has been submmited by CPU+ */+ virtual common::Status OnRunEnd() { return Status::OK(); }++ /**+ Called when session creation is complete+ This provides an opportunity for execution providers to optionally synchronize and+ clean up its temporary resources to reduce memory and ensure the first run is fast.+ */+ virtual common::Status OnSessionInitializationEnd() { return Status::OK(); }++ void InsertAllocator(AllocatorPtr allocator);+ void ReplaceAllocator(AllocatorPtr allocator);++ // creation of a fused node is not supported in a minimal build, so any EP enabled in that scenario must support+ // compilation via GraphViewer instances.+#if !defined(ORT_MINIMAL_BUILD)+ /**+ Given a list of fused_node, return create_state/compute/release_state func for each node.+ */+ virtual common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,+ std::vector<NodeComputeInfo>& node_compute_funcs);++ /**+ Given a list of fused_node, return a dll that expose functions for each node.+ For each node, there should be three symbols:+ Create_State_${node_name}+ Compute_${node_name}+ Release_State_${node_name}+ */+ virtual common::Status Compile(const std::vector<onnxruntime::Node*>& fused_nodes,+ std::string& dll_path);++#endif++#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)+ struct FusedNodeAndGraph {+ const std::reference_wrapper<onnxruntime::Node> fused_node;+ // GraphViewer that filters the full graph to the nodes that are covered by 'node'+ const std::reference_wrapper<GraphViewer> filtered_graph;+ };++ /**+ Given a collection of fused Nodes and the respective GraphViewer instance for the nodes that were fused,+ return create_state/compute/release_state func for each node.+ @remarks This is an optional interface that is only needed if the execution provider compiles nodes+ in a scenario involving the minimal build. i.e. on a mobile or embedded device with ORT format model.++ Do NOT cache the GraphViewer in FusedNodeAndGraph.filtered_graph in any of the NodeComputeInfo functions+ as it is only valid for the duration of the call to Compile.+ */+ virtual common::Status Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,+ std::vector<NodeComputeInfo>& node_compute_funcs);+#endif++ // Fusion approach that is suppported+ enum class FusionStyle {+ // The node fusion will create an onnxruntime::Function based Node that contains a completely new Graph instance+ // in the Node body. The original nodes and initializers are copied to the new Graph instance in Function::Body().+ // A GraphProto can be produced from the Node body.+ Function,++ // The node fusion will create a new Node that defines the inputs and outputs using the IndexedSubGraph+ // that GetCapability returned. The Node will not be onnxruntime::Function based so will have no Body().+ // Instead a GraphViewer that filters the full Graph to the fused Nodes will be created.+ // This is significantly cheaper as it doesn't incur the cost of creating a new Graph instance,+ // and can be supported in a minimal build.+ FilteredGraphViewer+ };++ virtual FusionStyle GetFusionStyle() const {+ // existing EPs use this mode so default to it.+ // newer EPs that can use the cheaper approach, or need to run in a minimal build, should override to return+ // FilteredGraphViewer+ return FusionStyle::Function;+ }++ void SetLogger(const logging::Logger* logger) {+ logger_ = logger;+ }++ const logging::Logger* GetLogger() const {+ return logger_;+ }++ /** Generate a unique id that can be used in a MetaDef name. Values are unique for a model instance.+ The model hash is also returned if you wish to include that in the MetaDef name to ensure uniqueness across models.+ @param graph_viewer[in] Graph viewer that GetCapability was called with. Can be for the main graph or nested graph.+ @param model_hash[out] Returns the hash for the main (i.e. top level) graph in the model.+ This is created using the model path if available,+ or the model input names and the output names from all nodes in the main graph.+ @remarks e.g. the TensorRT Execution Provider is used in multiple sessions and the underlying infrastructure caches+ compiled kernels, so the name must be unique and deterministic across models and sessions.+ NOTE: Ideally this would be a protected method, but to work across the EP bridge it has to be public and+ virtual, and ModelMetadefIdGenerator but be defined in the header as well.+ */+ virtual int GenerateMetaDefId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash) const;++ private:+ const std::string type_;+ AllocatorMap allocators_;+ MemoryInfoSet mem_info_set_; // to ensure only allocators with unique OrtMemoryInfo are registered in the provider.+ //It will be set when this object is registered to a session+ const logging::Logger* logger_ = nullptr;+ // convenience list of the allocators so GetAllocatorList doesn't have to build a new vector each time+ // contains the same instances as allocators_+ std::vector<AllocatorPtr> allocator_list_;++ // helper to generate ids that are unique to model and deterministic, even if the execution provider is shared across+ // multiple sessions.+ class ModelMetadefIdGenerator {+ public:+ int GenerateId(const onnxruntime::GraphViewer& graph_viewer, uint64_t& model_hash);++ private:+ std::unordered_map<uint64_t, int64_t> main_graph_hash_; // map graph instance hash to model contents hash+ std::unordered_map<int64_t, int> model_metadef_id_; // current unique id for model+ };++ std::unique_ptr<ModelMetadefIdGenerator> metadef_id_generator_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/fence.h b/include/onnxruntime/core/framework/fence.hnew file mode 100644index 0000000..e8408d1--- /dev/null+++ b/include/onnxruntime/core/framework/fence.h@@ -0,0 +1,56 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {++using ProviderType = const std::string&;++/*+ We use a simple fence mechanism for async compute. Assumptions in this fence mechanism:+ * Execution provider command queues, which execute in the same order of submit+ * No fence needed for kernels within one execution provider command queue+ * Fence is used to synchronize between command queues, and execution providers++ Fence usage:+ 1. Fence object would be created by allocation planer for input/output when KernelDef::ExecQueueId() is not zero+ 2. If fence object exists, executor would call BeforeUsingAs* prior to kernel::Compute(), and AfterUsedAs* afterwards+*/+class IFence {+ public:+ virtual ~IFence() = default;++ /**+ Called by executor before OrtValue is used as input in a compute kernel in provider_type and exec queue_id+ This should wait in the specified provider's exec queue for previous write to OrtValue to finish+ */+ virtual void BeforeUsingAsInput(onnxruntime::ProviderType provider_type, int queue_id) = 0;++ /**+ Called by executor before OrtValue is used as output in a compute kernel in provider_type and exec queue_id+ This should wait in the specified provider's exec queue for previous read to OrtValue to finish+ */+ virtual void BeforeUsingAsOutput(onnxruntime::ProviderType provider_type, int queue_id) = 0;++ /**+ Called by executor after OrtValue is used as input in a compute kernel in provider_type and exec queue_id+ This should update the read fence of the MLValue+ */+ virtual void AfterUsedAsInput(int queue_id) = 0;++ /**+ Called by executor after OrtValue is used as output in a compute kernel in provider_type and exec queue_id+ This should update the write fence of the MLValue+ */+ virtual void AfterUsedAsOutput(int queue_id) = 0;++ /**+ Called by executor before release OrtValue to see whether async data read is finished or not. This is non-blocking.+ */+ virtual bool CanRelease() = 0;+};+using Fence_t = IFence*;+using FencePtr = std::shared_ptr<IFence>;++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/framework_common.h b/include/onnxruntime/core/framework/framework_common.hnew file mode 100644index 0000000..dd0dea9--- /dev/null+++ b/include/onnxruntime/core/framework/framework_common.h@@ -0,0 +1,22 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <string>+#include <unordered_map>+#include <vector>+#include "run_options.h"++namespace onnxruntime { // forward declarations+class Model;+class GraphTransformer;+class NodeArg;+} // namespace onnxruntime++namespace onnxruntime {+using InputDefList = std::vector<const onnxruntime::NodeArg*>;+using OutputDefList = std::vector<const onnxruntime::NodeArg*>;++using NameMLValMap = std::unordered_map<std::string, OrtValue>;+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/func_api.h b/include/onnxruntime/core/framework/func_api.hnew file mode 100644index 0000000..93b52d8--- /dev/null+++ b/include/onnxruntime/core/framework/func_api.h@@ -0,0 +1,27 @@+#pragma once+#include "core/common/status.h"+using onnxruntime::common::Status; // TODO: Needed by WinML, but shouldn't be put into the global namespace like this++namespace onnxruntime {++// AllocateFunc(void* handle, size_t alignment, size_t size)+using AllocateFunc = void* (*)(void*, size_t, size_t);+using DestroyFunc = void (*)(void*, void*);+using AllocatorHandle = void*;++typedef struct {+ //right now we only include allocation for host memory+ AllocateFunc allocate_func;+ DestroyFunc release_func;+ AllocatorHandle allocator_handle;+ const char* node_name;+} ComputeContext;++using FunctionState = void*;+// take the ComputeContext, and create a function state.+using CreateFunctionStateC = int (*)(ComputeContext*, FunctionState*);+// pass in the function state and input/output tensors, perform compute and return status+using ComputeFuncC = common::Status (*)(FunctionState, const OrtApi*, OrtKernelContext*);+// release the function state.+using DestroyFunctionStateC = void (*)(FunctionState);+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.hnew file mode 100644index 0000000..63c9269--- /dev/null+++ b/include/onnxruntime/core/framework/kernel_def_builder.h@@ -0,0 +1,339 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <memory>+#include <string>+#include <unordered_map>+#include <vector>+#include <limits.h>++#include "core/common/common.h"+#include "core/common/optional.h"+#include "core/graph/basic_types.h"+#include "core/framework/data_types.h"+#include "core/framework/allocator.h"++namespace onnxruntime {+class KernelDefBuilder;++typedef std::map<size_t, OrtMemType> MemTypeMap;++class KernelDef {+ private:+ // note that input/output might be on CPU implicitly when the node is from CPU execution provider+ static inline bool MemTypeOnCpuExplicitly(OrtMemType mem_type) {+ return mem_type == OrtMemTypeCPUInput || mem_type == OrtMemTypeCPUOutput;+ }++ public:+ explicit KernelDef() = default;++ const std::string& OpName() const {+ return op_name_;+ }++ const std::string& Domain() const {+ return op_domain_;+ }++ void SinceVersion(/*out*/ int* start, /*out*/ int* end) const {+ *start = op_since_version_start_;+ *end = op_since_version_end_;+ }++#ifdef onnxruntime_PYBIND_EXPORT_OPSCHEMA+ const std::pair<int, int> SinceVersion() const {+ return std::pair<int, int>(op_since_version_start_, op_since_version_end_);+ }+#endif++ onnxruntime::ProviderType Provider() const {+ return provider_type_;+ }++ const std::map<std::string, std::vector<MLDataType>>& TypeConstraints() const {+ return type_constraints_;+ }++ const std::vector<std::pair<int, int>>& MayInplace() const {+ return inplace_map_;+ }++ const std::vector<std::pair<int, int>>& Alias() const {+ return alias_map_;+ }++ const optional<std::pair<int, int>>& VariadicAlias() const {+ return variadic_alias_offsets_;+ }++ OrtMemType InputMemoryType(size_t input_index) const {+ auto it = input_memory_type_args_.find(input_index);+ if (it == input_memory_type_args_.end())+ return default_inputs_mem_type_;+ return it->second;+ }++ bool IsInputOnCpu(size_t input_index) const { return MemTypeOnCpuExplicitly(InputMemoryType(input_index)); }++ bool IsOutputOnCpu(size_t output_index) const { return MemTypeOnCpuExplicitly(OutputMemoryType(output_index)); }++ bool AllocateInputsContiguously() const { return allocate_inputs_contiguously_; }++ OrtMemType OutputMemoryType(size_t output_index) const {+ auto it = output_memory_type_args_.find(output_index);+ if (it == output_memory_type_args_.end())+ return default_outputs_mem_type_;+ return it->second;+ }++ int ExecQueueId() const {+ return exec_queue_id_;+ }++ bool IsConflict(const KernelDef& other) const;++ uint64_t GetHash() const noexcept {+ // if we need to support different hash versions we can update CalculateHash to take a version number+ // and calculate any non-default versions dynamically. we only use this during kernel lookup so+ // it's not performance critical+ return hash_;+ }++ private:+ friend class KernelDefBuilder;++ // called once by KernelDefBuilder::Build+ void CalculateHash();++ // The operator name supported by <*this> kernel..+ std::string op_name_;++ // The operator since_version range supported by <*this> kernel.+ // A kernel could support an operator definition between <op_since_version_start>+ // and <op_since_version_end> (inclusive).+ int op_since_version_start_ = 1;+ int op_since_version_end_ = INT_MAX;++ // The operator domain supported by <*this> kernel.+ // Default to 'onnxruntime::kOnnxDomain'.+ // Please note the behavior of std::string("") and std::string() are different+ std::string op_domain_;++ // The type of the execution provider.+ std::string provider_type_;++ // The supported data types for inputs/outputs.+ // Key is input/output name defined in op schema, Value are supported types.+ // note: std::map as we need the order to be deterministic for the hash+ std::map<std::string, std::vector<MLDataType>> type_constraints_;++ // An element <i, j> means that output j reuses the memory of input i.+ std::vector<std::pair<int, int>> inplace_map_;++ // An element <i, j> means that output j is an alias of input i.+ std::vector<std::pair<int, int>> alias_map_;++ // This variable stores <input_offset, output_offset> for the variadic alias mapping+ // output 'i + output_offset' is an alias of input 'i + input_offset' for all i >= 0+ optional<std::pair<int, int>> variadic_alias_offsets_;++ // Require input tensors to be allocated contiguously.+ bool allocate_inputs_contiguously_ = false;++ // The memory types of inputs/outputs of this kernel+ MemTypeMap input_memory_type_args_;+ MemTypeMap output_memory_type_args_;++ // execution command queue id, 0 for default queue in execution provider+ int exec_queue_id_ = 0;+ // Default memory type for all inputs+ OrtMemType default_inputs_mem_type_{OrtMemTypeDefault};+ // Default memory type for all outputs+ OrtMemType default_outputs_mem_type_{OrtMemTypeDefault};++ // hash of kernel definition for lookup in minimal build+ uint64_t hash_ = 0;+};++class KernelDefBuilder {+ public:+ explicit KernelDefBuilder()+ : kernel_def_(new KernelDef()) {}++ KernelDefBuilder& SetName(const std::string& op_name);+ KernelDefBuilder& SetName(const char* op_name);++ KernelDefBuilder& SetDomain(const std::string& domain);+ KernelDefBuilder& SetDomain(const char* domain);++ /**+ This kernel supports operator definition since <since_version> (to latest).+ */+ KernelDefBuilder& SinceVersion(int since_version) {+ kernel_def_->op_since_version_start_ = since_version;+ return *this;+ }++ /**+ The start and end version should be set accordingly per version range for+ each domain registered in OpSchemaRegistry::DomainToVersionRange in+ \onnxruntime\onnxruntime\core\graph\op.h as below.+ Key: domain. Value: <lowest version, highest version> pair.+ std::unordered_map<std::string, std::pair<int, int>> map_;+ */+ KernelDefBuilder& SinceVersion(int since_version_start, int since_version_end) {+ kernel_def_->op_since_version_start_ = since_version_start;+ kernel_def_->op_since_version_end_ = since_version_end;+ return *this;+ }++ /**+ The execution provider type of the kernel.+ */+ KernelDefBuilder& Provider(onnxruntime::ProviderType provider_type);+ KernelDefBuilder& Provider(const char* provider_type);++ /**+ Specify the set of types that this kernel supports. A further restriction+ of the set of types specified in the op schema.+ The arg name could be either op formal parameter name, say "X", or type+ argument name specified in op schema, say "T".+ */+ KernelDefBuilder& TypeConstraint(const std::string& arg_name,+ const std::vector<MLDataType>& supported_types);+ KernelDefBuilder& TypeConstraint(const char* arg_name,+ const std::vector<MLDataType>& supported_types);++ /**+ Like TypeConstraint but supports just a single type.+ */+ KernelDefBuilder& TypeConstraint(const std::string& arg_name, MLDataType supported_type);+ KernelDefBuilder& TypeConstraint(const char* arg_name, MLDataType supported_type);++ /**+ Inplace mapping from inputs to outputs allowed.+ It means that uplayer runtime could do memory in-place optimization+ as it will not impact the correctness of this kernel.+ */+ KernelDefBuilder& MayInplace(const std::vector<std::pair<int, int>>& inplaces);+ KernelDefBuilder& MayInplace(int input_index, int output_index);++ /**+ Alias mapping from inputs to outputs. Different from Inplace that the+ content of the tensor is not changed. This is to take care of operators+ such as Identity and Reshape.+ */+ KernelDefBuilder& Alias(const std::vector<std::pair<int, int>>& aliases);+ KernelDefBuilder& Alias(int input_index, int output_index);++ /**+ Apply variadic number of alias mapping from inputs to outputs.+ This is effectively applying Alias(i + input_offset, i + output_offset) for i >= 0+ */+ KernelDefBuilder& VariadicAlias(int input_offset, int output_offset);++ /**+ Specify that this kernel requires input tensors to be allocated+ contiguously. This allows kernels to execute as a single large+ computation, rather than numerous smaller computations.+ */+ KernelDefBuilder& AllocateInputsContiguously() {+ kernel_def_->allocate_inputs_contiguously_ = true;+ return *this;+ }++ /**+ Specify that this kernel requires an input arg+ in certain memory type (instead of the default, device memory).+ */+ template <OrtMemType T>+ KernelDefBuilder& InputMemoryType(int input_index) {+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T));+ return *this;+ }++ KernelDefBuilder& InputMemoryType(OrtMemType type, int input_index) {+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, type));+ return *this;+ }++ /**+ Specify that this kernel requires input arguments+ in certain memory type (instead of the default, device memory).+ */+ template <OrtMemType T>+ KernelDefBuilder& InputMemoryType(const std::vector<int>& input_indexes) {+ for (auto input_index : input_indexes) {+ kernel_def_->input_memory_type_args_.insert(std::make_pair(input_index, T));+ }+ return *this;+ }++ /**+ Specify that this kernel provides an output arg+ in certain memory type (instead of the default, device memory).+ */+ template <OrtMemType T>+ KernelDefBuilder& OutputMemoryType(int output_index) {+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T));+ return *this;+ }++ KernelDefBuilder& OutputMemoryType(OrtMemType type, int output_index) {+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, type));+ return *this;+ }++ /**+ Specify that this kernel provides an output arguments+ in certain memory type (instead of the default, device memory).+ */+ template <OrtMemType T>+ KernelDefBuilder& OutputMemoryType(const std::vector<int>& output_indexes) {+ for (auto output_index : output_indexes) {+ kernel_def_->output_memory_type_args_.insert(std::make_pair(output_index, T));+ }+ return *this;+ }++ /**+ Specify that this kernel runs on which execution queue in the provider+ */+ KernelDefBuilder& ExecQueueId(int queue_id) {+ kernel_def_->exec_queue_id_ = queue_id;+ return *this;+ }++ /**+ Specify the default inputs memory type, if not specified, it is DefaultMemory+ */+ KernelDefBuilder& SetDefaultInputsMemoryType(OrtMemType mem_type) {+ kernel_def_->default_inputs_mem_type_ = mem_type;+ return *this;+ }++ /**+ Specify the default outputs memory type, if not specified, it is DefaultMemory+ */+ KernelDefBuilder& SetDefaultOutputMemoryType(OrtMemType mem_type) {+ kernel_def_->default_outputs_mem_type_ = mem_type;+ return *this;+ }++ /**+ Return the kernel definition, passing ownership of the KernelDef to the caller+ */+ std::unique_ptr<KernelDef> Build() {+ kernel_def_->CalculateHash();+ return std::move(kernel_def_);+ }++ private:+ // we own the KernelDef until Build() is called.+ std::unique_ptr<KernelDef> kernel_def_;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/kernel_registry.h b/include/onnxruntime/core/framework/kernel_registry.hnew file mode 100644index 0000000..0542183--- /dev/null+++ b/include/onnxruntime/core/framework/kernel_registry.h@@ -0,0 +1,95 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/framework/op_kernel.h"++namespace onnxruntime {+/**+ * Each provider has a KernelRegistry. Often, the KernelRegistry only belongs to that specific provider.+ *+ */+class KernelRegistry {+ public:+ KernelRegistry() = default;++ // Register a kernel with kernel definition and function to create the kernel.+ Status Register(KernelDefBuilder& kernel_def_builder, const KernelCreateFn& kernel_creator) ORT_MUST_USE_RESULT;++ Status Register(KernelCreateInfo&& create_info) ORT_MUST_USE_RESULT;++#if !defined(ORT_MINIMAL_BUILD)+ static bool HasImplementationOf(const KernelRegistry& r, const onnxruntime::Node& node,+ onnxruntime::ProviderType exec_provider) {+ const KernelCreateInfo* info;+ Status st = r.TryFindKernel(node, exec_provider, &info);+ return st.IsOK();+ }++ // factory functions should always return a unique_ptr for maximum flexibility+ // for its clients unless the factory is managing the lifecycle of the pointer+ // itself.+ // TODO(Task:132) Make usage of unique_ptr/shared_ptr as out param consistent+ Status TryCreateKernel(const onnxruntime::Node& node, const IExecutionProvider& execution_provider,+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors,+ const OrtValueNameIdxMap& mlvalue_name_idx_map, const FuncManager& funcs_mgr,+ const DataTransferManager& data_transfer_mgr,+ std::unique_ptr<OpKernel>& op_kernel) const ORT_MUST_USE_RESULT;++ // Check if an execution provider can create kernel for a node and return the kernel if so+ Status TryFindKernel(const onnxruntime::Node& node, onnxruntime::ProviderType exec_provider,+ const KernelCreateInfo** out) const;++#endif++ // Check if an execution provider can create kernel for a node and return the kernel if so.+ // Kernel matching is via kernel_def_hash.+ Status TryFindKernel(const onnxruntime::Node& node, onnxruntime::ProviderType exec_provider,+ uint64_t kernel_def_hash,+ const KernelCreateInfo** out) const;++ bool IsEmpty() const { return kernel_creator_fn_map_.empty(); }++#ifdef onnxruntime_PYBIND_EXPORT_OPSCHEMA+ // This is used by the opkernel doc generator to enlist all registered operators for a given provider's opkernel+ const KernelCreateMap& GetKernelCreateMap() const {+ return kernel_creator_fn_map_;+ }+#endif++ private:+#if !defined(ORT_MINIMAL_BUILD)+ // Check whether the types of inputs/outputs of the given node match the extra+ // type-constraints of the given kernel. This serves two purposes: first, to+ // select the right kernel implementation based on the types of the arguments+ // when we have multiple kernels, e.g., Clip<float> and Clip<int>; second, to+ // accommodate (and check) mapping of ONNX (specification) type to the onnxruntime+ // implementation type (e.g., if we want to implement ONNX's float16 as a regular+ // float in onnxruntime). (The second, however, requires a globally uniform mapping.)+ //+ // Note that this is not intended for type-checking the node against the ONNX+ // type specification of the corresponding op, which is done before this check.+ //+ // if this function is called before graph partition, then node.provider is not set.+ // In this case, kernel_def.provider must equal to exec_provider+ // otherwise, kernel_def.provider must equal to node.provider. exec_provider is ignored.+ static bool VerifyKernelDef(const onnxruntime::Node& node,+ const KernelDef& kernel_def,+ std::string& error_str);+#endif++ static std::string GetMapKey(const std::string& op_name, const std::string& domain, const std::string& provider) {+ std::string key(op_name);+ key.append(1, ' ').append(domain.empty() ? kOnnxDomainAlias : domain).append(1, ' ').append(provider);+ return key;+ }++ static std::string GetMapKey(const KernelDef& kernel_def) {+ return GetMapKey(kernel_def.OpName(), kernel_def.Domain(), kernel_def.Provider());+ }+ // Kernel create function map from op name to kernel creation info.+ // key is opname+domain_name+provider_name+ KernelCreateMap kernel_creator_fn_map_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/ml_value.h b/include/onnxruntime/core/framework/ml_value.hnew file mode 100644index 0000000..27ffa58--- /dev/null+++ b/include/onnxruntime/core/framework/ml_value.h@@ -0,0 +1,122 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <string>+#include "core/common/common.h"+#include "core/common/exceptions.h"+#include "core/framework/allocator.h"+#include "core/framework/data_types.h"+#include "core/framework/tensor.h"++namespace onnxruntime {+class SparseTensor;+class TensorSeq;+} // namespace onnxruntime++/**+ Represents both tensors and non-tensors.+*/+struct OrtValue {+ public:+ OrtValue() : data_(nullptr) {}+ ~OrtValue() = default;++ OrtValue(void* pData, onnxruntime::MLDataType type, onnxruntime::DeleteFunc deleter) {+ Init(pData, type, deleter);+ }++ void Init(void* pData, onnxruntime::MLDataType type, onnxruntime::DeleteFunc deleter) {+ data_.reset(pData, deleter);+ type_ = type;+ }++ bool IsAllocated() const {+ return data_ && type_;+ }++ template <typename T>+ const T& Get() const {+ ORT_ENFORCE(onnxruntime::DataTypeImpl::GetType<T>() == type_, onnxruntime::DataTypeImpl::GetType<T>(), " != ", type_);+ return *static_cast<T*>(data_.get());+ }++ template <typename T>+ T* GetMutable() {+ ORT_ENFORCE(onnxruntime::DataTypeImpl::GetType<T>() == type_, onnxruntime::DataTypeImpl::GetType<T>(), " != ", type_);+ return static_cast<T*>(data_.get());+ }++ bool IsTensor() const noexcept {+ return (type_ != nullptr && type_->IsTensorType());+ }++ bool IsTensorSequence() const noexcept {+ return (type_ != nullptr && type_->IsTensorSequenceType());+ }++ bool IsSparseTensor() const noexcept {+ return (type_ != nullptr && type_->IsSparseTensorType());+ }++ onnxruntime::MLDataType Type() const {+ return type_;+ }++ onnxruntime::Fence_t Fence() const {+ return fence_.get();+ }++ void SetFence(onnxruntime::FencePtr fence) {+ fence_ = fence;+ }++ void ShareFenceWith(OrtValue& v) {+ fence_ = v.fence_;+ }++ private:+ std::shared_ptr<void> data_;+ onnxruntime::MLDataType type_{nullptr};+ onnxruntime::FencePtr fence_;+};++template <>+inline const onnxruntime::Tensor& OrtValue::Get<onnxruntime::Tensor>() const {+ ORT_ENFORCE(IsTensor(), "Trying to get a Tensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return *static_cast<onnxruntime::Tensor*>(data_.get());+}++template <>+inline onnxruntime::Tensor* OrtValue::GetMutable<onnxruntime::Tensor>() {+ ORT_ENFORCE(IsTensor(), "Trying to get a Tensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return static_cast<onnxruntime::Tensor*>(data_.get());+}++template <>+inline const onnxruntime::TensorSeq& OrtValue::Get<onnxruntime::TensorSeq>() const {+ ORT_ENFORCE(IsTensorSequence(), "Trying to get a TensorSeq, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return *static_cast<onnxruntime::TensorSeq*>(data_.get());+}++template <>+inline onnxruntime::TensorSeq* OrtValue::GetMutable<onnxruntime::TensorSeq>() {+ ORT_ENFORCE(IsTensorSequence(), "Trying to get a TensorSeq, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return static_cast<onnxruntime::TensorSeq*>(data_.get());+}++template <>+inline const onnxruntime::SparseTensor& OrtValue::Get<onnxruntime::SparseTensor>() const {+ ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return *static_cast<onnxruntime::SparseTensor*>(data_.get());+}++template <>+inline onnxruntime::SparseTensor* OrtValue::GetMutable<onnxruntime::SparseTensor>() {+ ORT_ENFORCE(IsSparseTensor(), "Trying to get a SparseTensor, but got: ", onnxruntime::DataTypeImpl::ToString(type_));+ return static_cast<onnxruntime::SparseTensor*>(data_.get());+}++//TODO: remove the following line+#define MLValue OrtValuediff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.hnew file mode 100644index 0000000..2e3937a--- /dev/null+++ b/include/onnxruntime/core/framework/op_kernel.h@@ -0,0 +1,484 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <functional>++#include "core/common/exceptions.h"+#include "core/common/logging/logging.h"+#include "core/common/status.h"+#include "core/framework/execution_provider.h"+#include "core/framework/kernel_def_builder.h"+#include "core/framework/ml_value.h"+#include "core/framework/op_kernel_info.h"+#include "core/framework/op_node_proto_helper.h"+#include "core/framework/tensor.h"+#include "core/framework/sparse_tensor.h"+#include "core/graph/constants.h"+#include "core/graph/graph_viewer.h"+#include "core/graph/onnx_protobuf.h"+#include "gsl/gsl"++namespace onnxruntime {+class IExecutionFrame;+class OpKernelContext;+class OpKernelWrapper;+namespace concurrency {+class ThreadPool;+}++class OpKernel {+ public:+ using DoneCallback = std::function<void()>;++ explicit OpKernel(const OpKernelInfo& info) : op_kernel_info_(info) {}+ virtual ~OpKernel() = default;++ const onnxruntime::Node& Node() const {+ return op_kernel_info_.node();+ }++ const onnxruntime::KernelDef& KernelDef() const {+ return op_kernel_info_.GetKernelDef();+ }++ virtual Status Compute(_Inout_ OpKernelContext* context) const ORT_MUST_USE_RESULT = 0;++ virtual Status ComputeAsync(_Inout_ OpKernelContext*, DoneCallback) const ORT_MUST_USE_RESULT {+ ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");+ }++ // Override this function to PrePack initialized constant tensor to the format as needed.+ // For example, MatMul kernel can pack the input B if it is constant like code below.+ // Status PrePack(const Tensor& tensor, int input_idx, bool& is_packed) override {+ // is_packed = false;+ // if (input_idx == 1) {+ // this.Pack(tensor, this.buffer_);+ // is_packed = true;+ // }+ // return Status::OK();+ // }+ // Please refer to MatMulIntegerToFloatBase for a complete example+ // @param tesnor: The initialized constant tensor+ // @param input_idx: The input index of the tensor in this kernel+ // @param is_packed: Set it to true if the kernel packed the tensor or to false+ // The kernel is responsible keep the packed data and related metadata if is_packed is set to true+ // And the original intialized constant tensor will be released and not accessible anymore in Compute function.+ virtual Status PrePack(const Tensor& /*tensor*/, int /*input_idx*/, bool& is_packed) {+ is_packed = false;+ return Status::OK();+ }++ const OrtMemoryInfo& Allocator(int id, OrtMemType mem_type) const {+ return op_kernel_info_.GetMemoryInfo(id, mem_type);+ }++ const OpKernelInfo& Info() const { return op_kernel_info_; }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OpKernel);+ OpKernelInfo op_kernel_info_;+};++class OpKernelContext {+ public:+ using ArgMap = std::unordered_map<std::string, size_t>;++ OpKernelContext(_Inout_ IExecutionFrame* frame, _In_ const OpKernel* kernel,+ _In_opt_ concurrency::ThreadPool* threadpool, _In_ const logging::Logger& logger);++ virtual ~OpKernelContext() = default;++ /**+ Return the number of inputs for a variadic argument.+ @param arg_num The operator argument number.+ @returns Number of inputs the argument has.+ */+ int NumVariadicInputs(size_t arg_num) const;++ MLDataType InputType(int index) const;+ MLDataType OutputType(int index) const;++ template <typename T>+ const T* Input(int index) const {+ const OrtValue* p_ml_value = GetInputMLValue(index);+ ORT_TRY {+ return p_ml_value ? &(p_ml_value->Get<T>()) : nullptr;+ }+ ORT_CATCH(const std::exception& /*e*/) {+ ORT_THROW("Missing Input: " + kernel_->Node().InputDefs()[index]->Name());+ }+ }++ // Fetch a required input, enforcing that it is present.+ template <typename T>+ const T& RequiredInput(int index) const {+ const T* input_ptr = Input<T>(index);+ ORT_ENFORCE(input_ptr, "Required input at index ", index, " is not present.");+ return *input_ptr;+ }++ // Fetch output (non-tensor) with specified index.+ template <typename T>+ T* Output(int index) {+ if (index < 0 || index >= OutputCount())+ return nullptr;++ OrtValue* p_ml_value = GetOrCreateOutputMLValue(index);+ return p_ml_value ? p_ml_value->GetMutable<T>() : nullptr;+ }++ // In the case that memory allocation has not been done for an output tensor,+ // The memory allocation will be done on-the-fly with given tensor shape.+ // Return nullptr if the output is an unused optional output.+ Tensor* Output(int index, const TensorShape& shape);+ Tensor* Output(int index, const std::vector<int64_t>& shape);+ Tensor* Output(int index, const std::initializer_list<int64_t>& shape);++ // Fetch a required tensor output, enforcing that it is present.+ Tensor& RequiredOutput(int index, const TensorShape& shape) {+ Tensor* output_ptr = Output(index, shape);+ ORT_ENFORCE(output_ptr, "Required output at index ", index, " is not present.");+ return *output_ptr;+ }++ // Fetch a sparse-tensor output corresponding to the specified index.+ // num_values must specify the number of non-zero values (commonly known as NNZ/nnz),+ // and shape must specify the shape of the underlying dense-tensor.+ // Memory allocation for the output may happen when this method is invoked,+ // unless static optimization pre-allocates it.+ SparseTensor* Output(int index, size_t num_values, const TensorShape& shape);++ // Retrieve indexed shape obtained from memory planning before actual+ // computation. If the indexed shape cannot be inferred, this function returns+ // false.+ bool TryGetInferredInputShape(int index, TensorShape& shape) const;++ // Retrieve indexed shape obtained from memory planning before actual+ // computation. If the indexed shape cannot be inferred, this function returns+ // false.+ bool TryGetInferredOutputShape(int index, TensorShape& shape) const;++ const logging::Logger& Logger() const {+ return *logger_;+ }++ // always >= 0+ int InputCount() const {+ return static_cast<int>(kernel_->Node().InputDefs().size());+ }++ // always >= 0+ int ImplicitInputCount() const {+ return static_cast<int>(kernel_->Node().ImplicitInputDefs().size());+ }++ // always >= 0+ int OutputCount() const {+ return static_cast<int>(kernel_->Node().OutputDefs().size());+ }++ /**+ Return an allocator on device 0, with memtype of OrtMemTypeDefault.+ @remarks Use SafeInt when calculating the size of memory to allocate using AllocatorPtr->Alloc.+ */+ Status GetTempSpaceAllocator(AllocatorPtr* output) const ORT_MUST_USE_RESULT;++ /**+ Return the fence of current node's input.+ @param index The index of the input.+ @returns Point to the Fence of the input OrtValue.+ It is null if the input OrtValue doesn't have fence or the input is optional.+ */+ Fence_t InputFence(int index) const;++ /**+ Return the fence of current node's implicit input.+ @param index The index of the implicit input.+ @returns Point to the Fence of the implicit input OrtValue.+ It is null if the input OrtValue doesn't have fence or the input is optional.+ */+ Fence_t ImplicitInputFence(int index) const;++ /**+ Return the fence of current node's output identifed by index.+ @param index The index of the output.+ @returns Point to the Fence of the output OrtValue.+ It is null if the output OrtValue doesn't have fence or the output is optional.+ */+ Fence_t OutputFence(int index) const;++ /**+ Return the device id that current kernel runs on.+ */+ int GetDeviceId() const {+ return kernel_->Info().GetExecutionProvider()->GetDeviceId();+ }++ /**+ Returns the opset domain of the underlying kernel+ **/+ const std::string& GetOpDomain() const;++ /**+ Returns the optype of the underlying kernel+ **/+ const std::string& GetOpType() const;++ /**+ Returns the node name of the underlying kernel+ **/+ const std::string& GetNodeName() const;++ /**+ Returns the intra-op threadpool, if available.+ */+ _Ret_maybenull_ onnxruntime::concurrency::ThreadPool* GetOperatorThreadPool() const { return threadpool_; }++ /**+ Returns whether deterministic computation is preferred.+ */+ virtual bool GetUseDeterministicCompute() const {+ return true;+ }++ protected:+ onnxruntime::NodeIndex GetNodeIndex() const;++ const OrtValue* GetInputMLValue(int index) const;+ const OrtValue* GetImplicitInputMLValue(int index) const;+ OrtValue* GetOutputMLValue(int index);++ // Creates the OrtValue* based on the shape, if it does not exist+ // The parameter nnz is used only for sparse-tensors and indicates the+ // number of non-zero values (the number of elements in the values buffer allocated).+ OrtValue* OutputMLValue(int index, const TensorShape& shape, size_t nnz = 0);++ private:+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(OpKernelContext);++ OrtValue* GetOrCreateOutputMLValue(int index);++ int GetInputArgIndex(int index) const;+ int GetImplicitInputArgIndex(int index) const;+ int GetOutputArgIndex(int index) const;++ IExecutionFrame* const execution_frame_;+ const OpKernel* const kernel_;+ concurrency::ThreadPool* const threadpool_;+ const logging::Logger* const logger_;++ // The argument starting index in ExecutionFrame.+ int node_input_start_index_{-1};+ int node_implicit_input_start_index_{-1};+ int node_output_start_index_{-1};+};++// Fetching output tensor without shape is not allowed except when it already exists+template <>+inline Tensor* OpKernelContext::Output<Tensor>(int index) {+ OrtValue* p_ml_value = GetOutputMLValue(index);+ ORT_ENFORCE(p_ml_value, "Please fetch output tensor with specified shape.");+ return p_ml_value->GetMutable<Tensor>();+}++using KernelCreateFn = std::function<OpKernel*(const OpKernelInfo& info)>;+using KernelCreatePtrFn = std::add_pointer<OpKernel*(const OpKernelInfo& info)>::type;++struct KernelCreateInfo {+ std::unique_ptr<KernelDef> kernel_def; // Owned and stored in the global kernel registry.+ KernelCreateFn kernel_create_func;+ Status status;++ KernelCreateInfo(std::unique_ptr<KernelDef> definition,+ KernelCreateFn create_func)+ : kernel_def(std::move(definition)),+ kernel_create_func(create_func) {}++ KernelCreateInfo(KernelCreateInfo&& other) noexcept+ : kernel_def(std::move(other.kernel_def)),+ kernel_create_func(std::move(other.kernel_create_func)) {}++ KernelCreateInfo() = default;+};++using KernelCreateMap = std::multimap<std::string, KernelCreateInfo>;++// Forward declarations for the non-specialized BuildKernelCreateInfo method.+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();++namespace ml {+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();+} // namespace ml++namespace contrib {+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();+} // namespace contrib++namespace featurizers {+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();+} // namespace featurizers++namespace contrib {+namespace cuda {+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();+} // namespace cuda+} // namespace contrib++namespace contrib {+namespace rocm {+template <typename T>+KernelCreateInfo BuildKernelCreateInfo();+} // namespace rocm+} // namespace contrib++using BuildKernelCreateInfoFn = KernelCreateInfo (*)();++// Naming convention for operator kernel classes+#define ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name) \+ provider##_##name##_##domain##_ver##ver++#define ONNX_CPU_OPERATOR_KERNEL(name, ver, builder, ...) \+ ONNX_OPERATOR_KERNEL_EX(name, kOnnxDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_ML_KERNEL(name, ver, builder, ...) \+ ONNX_OPERATOR_KERNEL_EX(name, kMLDomain, ver, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_OPERATOR_KERNEL_EX(name, domain, ver, provider, builder, ...) \+ class ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(provider, domain, ver, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(ver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++#define ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name) \+ provider##_##name##_##domain##_ver##startver##_##endver++#define ONNX_CPU_OPERATOR_VERSIONED_KERNEL(name, startver, endver, builder, ...) \+ ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kOnnxDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_VERSIONED_ML_KERNEL(name, startver, endver, builder, ...) \+ ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, kMLDomain, startver, endver, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_OPERATOR_VERSIONED_KERNEL_EX(name, domain, startver, endver, provider, builder, ...) \+ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(provider, domain, startver, endver, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(startver, endver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++#define ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name) \+ provider##_##name##_##domain##_ver##ver##_##type++#define ONNX_CPU_OPERATOR_TYPED_KERNEL(name, ver, type, builder, ...) \+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kOnnxDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_TYPED_ML_KERNEL(name, ver, type, builder, ...) \+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMLDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_TYPED_MS_KERNEL(name, ver, type, builder, ...) \+ ONNX_OPERATOR_TYPED_KERNEL_EX(name, kMSDomain, ver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_OPERATOR_TYPED_KERNEL_EX(name, domain, ver, type, provider, builder, ...) \+ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(ver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++#define ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name) \+ provider##_##name##_##domain##_ver##ver##_##type1##_##type2++#define ONNX_OPERATOR_TWO_TYPED_KERNEL_EX(name, domain, ver, type1, type2, provider, builder, ...) \+ class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, ver, type1, type2, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(ver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name) \+ provider##_##name##_##domain##_ver##startver##_##endver##_##type++#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(name, startver, endver, type, builder, ...) \+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kOnnxDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_ML_KERNEL(name, startver, endver, type, builder, ...) \+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMLDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_CPU_OPERATOR_VERSIONED_TYPED_MS_KERNEL(name, startver, endver, type, builder, ...) \+ ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, kMSDomain, startver, endver, type, kCpuExecutionProvider, builder, __VA_ARGS__)++#define ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(name, domain, startver, endver, type, provider, builder, ...) \+ class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(startver, endver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++#define ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name) \+ provider##_##name##_##domain##_ver##startver##_##endver##_##type1##_##type2++#define ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_EX(name, domain, startver, endver, type1, type2, provider, builder, ...) \+ class ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name); \+ template <> \+ KernelCreateInfo \+ BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(provider, domain, startver, endver, type1, type2, name)>() { \+ return KernelCreateInfo( \+ builder.SetName(#name) \+ .SetDomain(domain) \+ .SinceVersion(startver, endver) \+ .Provider(provider) \+ .Build(), \+ static_cast<KernelCreatePtrFn>([](const OpKernelInfo& info) -> OpKernel* { return new __VA_ARGS__(info); })); \+ }++// Use within macro definitions to create a custom vector of constraints.+// Example: #define REG_KERNEL(OP, VERSION, KERNEL_CLASS, Type, ...)+// .TypeConstraint("T", BuildKernelDefConstraints<Type, __VA_ARGS_>())+template <typename T, typename... Types>+inline std::vector<MLDataType> BuildKernelDefConstraints() {+ return {DataTypeImpl::GetTensorType<T>(), DataTypeImpl::GetTensorType<Types>()...};+}++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/op_kernel_info.h b/include/onnxruntime/core/framework/op_kernel_info.hnew file mode 100644index 0000000..40b8c9a--- /dev/null+++ b/include/onnxruntime/core/framework/op_kernel_info.h@@ -0,0 +1,66 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/framework/execution_provider.h"+#include "core/framework/kernel_def_builder.h"+#include "core/framework/ml_value.h"+#include "core/framework/op_node_proto_helper.h"+#include "core/graph/graph_viewer.h"+#include "gsl/gsl"++namespace onnxruntime {++class OrtValueNameIdxMap;+class FuncManager;+class DataTransferManager;++// A very light-weight class, which works as an aggregated+// view of all data needed for constructing a Kernel instance.+// NOTE: it does not own/hold any objects.+class OpKernelInfo : public OpNodeProtoHelper<ProtoHelperNodeContext> {+ public:+ explicit OpKernelInfo(const onnxruntime::Node& node,+ const KernelDef& kernel_def,+ const IExecutionProvider& execution_provider,+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors,+ const OrtValueNameIdxMap& mlvalue_name_idx_map,+ const FuncManager& funcs_mgr,+ const DataTransferManager& data_transfer_mgr);++ OpKernelInfo(const OpKernelInfo& other);++ const OrtMemoryInfo& GetMemoryInfo(int device_id, OrtMemType mem_type) const;++ AllocatorPtr GetAllocator(int device_id, OrtMemType mem_type) const;++ const KernelDef& GetKernelDef() const;++ const IExecutionProvider* GetExecutionProvider() const noexcept;++ const DataTransferManager& GetDataTransferManager() const noexcept;++ const onnxruntime::Node& node() const noexcept;++ bool TryGetConstantInput(int input_index, const Tensor** constant_input_value) const;++ common::Status GetFusedFuncs(NodeComputeInfo*& compute_info) const;++ private:+ ORT_DISALLOW_MOVE(OpKernelInfo);+ ORT_DISALLOW_ASSIGNMENT(OpKernelInfo);++ const onnxruntime::Node& node_;+ const KernelDef& kernel_def_;+ // For non cpu/cuda case, this pointer should be set so that function kernel+ // will delegate kernel compute call to <execution_provider> compute call.+ gsl::not_null<const ::onnxruntime::IExecutionProvider*> execution_provider_;+ const std::unordered_map<int, OrtValue>& constant_initialized_tensors_;+ const OrtValueNameIdxMap& ort_value_name_idx_map_;+ const FuncManager& funcs_mgr_;+ const DataTransferManager& data_transfer_mgr_;+ ProtoHelperNodeContext proto_helper_context_;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/op_node_proto_helper.h b/include/onnxruntime/core/framework/op_node_proto_helper.hnew file mode 100644index 0000000..7443459--- /dev/null+++ b/include/onnxruntime/core/framework/op_node_proto_helper.h@@ -0,0 +1,157 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/status.h"+#include "core/graph/graph_viewer.h"+#include "gsl/gsl"++#ifdef __has_attribute+#define ORT_HAVE_ATTRIBUTE(x) __has_attribute(x)+#else+#define ORT_HAVE_ATTRIBUTE(x) 0+#endif++#if ORT_HAVE_ATTRIBUTE(nodiscard)+#define MUST_USE_RESULT [[nodiscard]]+#elif defined(__clang__) && ORT_HAVE_ATTRIBUTE(warn_unused_result)+#define MUST_USE_RESULT __attribute__((warn_unused_result))+#else+#define MUST_USE_RESULT+#endif++class IMLOpKernel;++namespace onnxruntime {++/**+ A set of wrappers with common signatures for use with both OpKernelInfo+ (as its base class) and InferenceContext. Used by ABI kernels for both+ shape / type inference and kernel construction+*/+template <class Impl_t>+class OpNodeProtoHelper {+ public:+ explicit OpNodeProtoHelper(const Impl_t* impl) : impl_(impl) {}++ /**+ Get a single attribute+ Call this function for a required attribute or when a default value for an optional attribute is specified in the op schema+ */+ template <typename T>+ MUST_USE_RESULT Status GetAttr(const std::string& name, T* value) const;++ /**+ Get a single attribute+ Call this function only when a default value for an optional attribute isn't specified in the op schema+ */+ template <typename T>+ T GetAttrOrDefault(const std::string& name, const T& default_value) const {+ T tmp;+ return GetAttr<T>(name, &tmp).IsOK() ? tmp : default_value;+ }++ /**+ Get a single attribute+ Call this function only when a default value for an optional attribute isn't specified in the op schema+ */+ template <typename T>+ void GetAttrOrDefault(const std::string& name, T* value, const T& default_value) const {+ if (!GetAttr<T>(name, value).IsOK())+ *value = default_value;+ }++ /**+ Get repeated attributes+ Call this function only when a default value for an optional attribute isn't specified in the op schema+ */+ template <typename T>+ MUST_USE_RESULT std::vector<T> GetAttrsOrDefault(const std::string& name, const std::vector<T>& default_value = std::vector<T>{}) const {+ std::vector<T> tmp;+ return GetAttrs<T>(name, tmp).IsOK() ? tmp : default_value;+ }++ /**+ Get repeated attributes+ */+ template <typename T>+ MUST_USE_RESULT Status GetAttrs(const std::string& name, std::vector<T>& values) const;++ template <typename T>+ MUST_USE_RESULT Status GetAttrs(const std::string& name, gsl::span<T> values) const;++ /// <summary>+ /// Return a gsl::span that points to an array of primitive types held by AttributeProto+ /// This function allows to avoid copying big attributes locally into a kernel and operate on+ /// AttributeProto data directly.+ ///+ /// Does not apply to strings, Tensors and Sparse Tensors that require special treatment.+ /// </summary>+ /// <typeparam name="T">Primitive type contained in the array</typeparam>+ /// <param name="name">Attribute name</param>+ /// <param name="values">Attribute data in a span, out parameter</param>+ /// <returns>Status</returns>+ template <typename T>+ MUST_USE_RESULT Status GetAttrsAsSpan(const std::string& name, gsl::span<const T>& values) const;++ MUST_USE_RESULT Status GetAttrsStringRefs(const std::string& name,+ std::vector<std::reference_wrapper<const std::string>>& refs) const;++ uint32_t GetPrimitiveAttrElementCount(ONNX_NAMESPACE::AttributeProto_AttributeType type,+ const std::string& name) const noexcept;++ bool HasPrimitiveAttribute(ONNX_NAMESPACE::AttributeProto_AttributeType type,+ const std::string& name) const noexcept;++ uint32_t GetInputCount() const {+ return gsl::narrow_cast<uint32_t>(impl_->getNumInputs());+ }++ uint32_t GetOutputCount() const {+ return gsl::narrow_cast<uint32_t>(impl_->getNumOutputs());+ }++ const ONNX_NAMESPACE::TypeProto* GetInputType(size_t index) const {+ return impl_->getInputType(index);+ }++ const ONNX_NAMESPACE::TypeProto* GetOutputType(size_t index) const {+ // Work around lack of a const method from the onnx InferenceContext interface+ return const_cast<Impl_t*>(impl_)->getOutputType(index);+ }++ // Try to query an attribute, returning nullptr if it doesn't exist+ const ONNX_NAMESPACE::AttributeProto* TryGetAttribute(const std::string& name) const {+ return impl_->getAttribute(name);+ }++ const ONNX_NAMESPACE::AttributeProto* GetAttribute(const std::string& name) const {+ const ONNX_NAMESPACE::AttributeProto* attr = TryGetAttribute(name);+ ORT_ENFORCE(attr != nullptr);+ return attr;+ }++ private:+ OpNodeProtoHelper() = delete;+ const Impl_t* impl_ = nullptr;+};++// The methods on the following class are called by OpNodeProtoHelper, implementing+// the same signatures as InferenceContext other than const-ness.+class ProtoHelperNodeContext {+ public:+ explicit ProtoHelperNodeContext(const onnxruntime::Node& node) : node_(node) {}+ ProtoHelperNodeContext() = delete;++ const ONNX_NAMESPACE::AttributeProto* getAttribute(const std::string& name) const;+ size_t getNumInputs() const;+ const ONNX_NAMESPACE::TypeProto* getInputType(size_t index) const;+ size_t getNumOutputs() const;+ const ONNX_NAMESPACE::TypeProto* getOutputType(size_t index) const;++ private:+ const onnxruntime::Node& node_;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/ortdevice.h b/include/onnxruntime/core/framework/ortdevice.hnew file mode 100644index 0000000..afc1518--- /dev/null+++ b/include/onnxruntime/core/framework/ortdevice.h@@ -0,0 +1,72 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <sstream>++// Struct to represent a physical device.+struct OrtDevice {+ using DeviceType = int8_t;+ using MemoryType = int8_t;+ using DeviceId = int16_t;++ // Pre-defined device types.+ static const DeviceType CPU = 0;+ static const DeviceType GPU = 1; // Nvidia or AMD+ static const DeviceType FPGA = 2;++ struct MemType {+ // Pre-defined memory types.+ static const MemoryType DEFAULT = 0;+ static const MemoryType CUDA_PINNED = 1;+ static const MemoryType HIP_PINNED = 2;+ };++ constexpr OrtDevice(DeviceType device_type_, MemoryType memory_type_, DeviceId device_id_)+ : device_type(device_type_),+ memory_type(memory_type_),+ device_id(device_id_) {}++ constexpr OrtDevice() : OrtDevice(CPU, MemType::DEFAULT, 0) {}++ DeviceType Type() const {+ return device_type;+ }++ MemoryType MemType() const {+ return memory_type;+ }++ DeviceId Id() const {+ return device_id;+ }++ std::string ToString() const {+ std::ostringstream ostr;+ ostr << "Device:["+ << "DeviceType:" << static_cast<int>(device_type)+ << " MemoryType:" << static_cast<int>(memory_type)+ << " DeviceId:" << device_id+ << "]";+ return ostr.str();+ }++ private:+ // Device type.+ DeviceType device_type;++ // Memory type.+ MemoryType memory_type;++ // Device index.+ DeviceId device_id;+};++inline bool operator==(const OrtDevice& left, const OrtDevice& other) {+ return left.Id() == other.Id() && left.MemType() == other.MemType() && left.Type() == other.Type();+}++inline bool operator!=(const OrtDevice& left, const OrtDevice& other) {+ return !(left == other);+}diff --git a/include/onnxruntime/core/framework/ortmemoryinfo.h b/include/onnxruntime/core/framework/ortmemoryinfo.hnew file mode 100644index 0000000..77df903--- /dev/null+++ b/include/onnxruntime/core/framework/ortmemoryinfo.h@@ -0,0 +1,63 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++struct OrtMemoryInfo {+ OrtMemoryInfo() = default; // to allow default construction of Tensor++ // use string for name, so we could have customized allocator in execution provider.+ const char* name = nullptr;+ int id = -1;+ OrtMemType mem_type = OrtMemTypeDefault;+ OrtAllocatorType alloc_type = Invalid;+ OrtDevice device;++ constexpr OrtMemoryInfo(const char* name_, OrtAllocatorType type_, OrtDevice device_ = OrtDevice(), int id_ = 0,+ OrtMemType mem_type_ = OrtMemTypeDefault)+#if ((defined(__GNUC__) && __GNUC__ > 4) || defined(__clang__))+ // this causes a spurious error in CentOS gcc 4.8 build so disable if GCC version < 5+ __attribute__((nonnull))+#endif+ : name(name_),+ id(id_),+ mem_type(mem_type_),+ alloc_type(type_),+ device(device_) {+ }++ // To make OrtMemoryInfo become a valid key in std map+ bool operator<(const OrtMemoryInfo& other) const {+ if (alloc_type != other.alloc_type)+ return alloc_type < other.alloc_type;+ if (mem_type != other.mem_type)+ return mem_type < other.mem_type;+ if (id != other.id)+ return id < other.id;++ return strcmp(name, other.name) < 0;+ }++ std::string ToString() const {+ std::ostringstream ostr;+ ostr << "OrtMemoryInfo:["+ << "name:" << name+ << " id:" << id+ << " OrtMemType:" << mem_type+ << " OrtAllocatorType:" << alloc_type+ << " " << device.ToString()+ << "]";+ return ostr.str();+ }+};++inline bool operator==(const OrtMemoryInfo& left, const OrtMemoryInfo& other) {+ return left.mem_type == other.mem_type &&+ left.alloc_type == other.alloc_type &&+ left.id == other.id &&+ strcmp(left.name, other.name) == 0;+}++inline bool operator!=(const OrtMemoryInfo& lhs, const OrtMemoryInfo& rhs) { return !(lhs == rhs); }++std::ostream& operator<<(std::ostream& out, const OrtMemoryInfo& info);diff --git a/include/onnxruntime/core/framework/provider_options.h b/include/onnxruntime/core/framework/provider_options.hnew file mode 100644index 0000000..aab13e8--- /dev/null+++ b/include/onnxruntime/core/framework/provider_options.h@@ -0,0 +1,18 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <string>+#include <unordered_map>+#include <vector>++namespace onnxruntime {++// data types for execution provider options++using ProviderOptions = std::unordered_map<std::string, std::string>;+using ProviderOptionsVector = std::vector<ProviderOptions>;+using ProviderOptionsMap = std::unordered_map<std::string, ProviderOptions>;++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/provider_options_utils.h b/include/onnxruntime/core/framework/provider_options_utils.hnew file mode 100644index 0000000..e326481--- /dev/null+++ b/include/onnxruntime/core/framework/provider_options_utils.h@@ -0,0 +1,165 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <algorithm>+#include <functional>+#include <type_traits>+#include <unordered_map>+#include <vector>++#include "core/common/common.h"+#include "core/common/parse_string.h"+#include "core/framework/provider_options.h"++namespace onnxruntime {++template <typename TEnum>+using EnumNameMapping = std::vector<std::pair<TEnum, std::string>>;++/**+ * Given a mapping and an enumeration value, gets the corresponding name.+ */+template <typename TEnum>+Status EnumToName(const EnumNameMapping<TEnum>& mapping, TEnum value, std::string& name) {+ const auto it = std::find_if(+ mapping.begin(), mapping.end(),+ [&value](const std::pair<TEnum, std::string>& entry) {+ return entry.first == value;+ });+ ORT_RETURN_IF(+ it == mapping.end(),+ "Failed to map enum value to name: ", static_cast<typename std::underlying_type<TEnum>::type>(value));+ name = it->second;+ return Status::OK();+}++template <typename TEnum>+std::string EnumToName(const EnumNameMapping<TEnum>& mapping, TEnum value) {+ std::string name;+ ORT_THROW_IF_ERROR(EnumToName(mapping, value, name));+ return name;+}++/**+ * Given a mapping and a name, gets the corresponding enumeration value.+ */+template <typename TEnum>+Status NameToEnum(+ const EnumNameMapping<TEnum>& mapping, const std::string& name, TEnum& value) {+ const auto it = std::find_if(+ mapping.begin(), mapping.end(),+ [&name](const std::pair<TEnum, std::string>& entry) {+ return entry.second == name;+ });+ ORT_RETURN_IF(+ it == mapping.end(),+ "Failed to map enum name to value: ", name);+ value = it->first;+ return Status::OK();+}++template <typename TEnum>+TEnum NameToEnum(const EnumNameMapping<TEnum>& mapping, const std::string& name) {+ TEnum value;+ ORT_THROW_IF_ERROR(NameToEnum(mapping, name, value));+ return value;+}++class ProviderOptionsParser {+ public:+ /**+ * Adds a parser for a particular provider option value.+ *+ * @param name The provider option name.+ * @param value_parser An object that parses the option value.+ * It should be callable with the following signature and return+ * whether the parsing was successful:+ * Status value_parser(const std::string&)+ *+ * @return The current ProviderOptionsParser instance.+ */+ template <typename ValueParserType>+ ProviderOptionsParser& AddValueParser(+ const std::string& name, ValueParserType value_parser) {+ ORT_ENFORCE(+ value_parsers_.emplace(name, ValueParser{value_parser}).second,+ "Provider option \"", name, "\" already has a value parser.");+ return *this;+ }++ /**+ * Adds a parser for a particular provider option value which converts a+ * value to the right type and assigns it to the given reference.+ *+ * IMPORTANT: This function stores a reference to the destination variable.+ * The caller must ensure that the reference is valid when Parse() is called!+ *+ * @param name The provider option name.+ * @param dest The destination variable reference.+ *+ * @return The current ProviderOptionsParser instance.+ */+ template <typename ValueType>+ ProviderOptionsParser& AddAssignmentToReference(+ const std::string& name, ValueType& dest) {+ return AddValueParser(+ name,+ [&dest](const std::string& value_str) -> Status {+ return ParseStringWithClassicLocale(value_str, dest);+ });+ }++ /**+ * Adds a parser for a particular provider option value which maps an+ * enumeration name to a value and assigns it to the given reference.+ *+ * IMPORTANT: This function stores references to the mapping and destination+ * variables. The caller must ensure that the references are valid when+ * Parse() is called!+ *+ * @param name The provider option name.+ * @param mapping The enumeration value to name mapping.+ * @param dest The destination variable reference.+ *+ * @return The current ProviderOptionsParser instance.+ */+ template <typename EnumType>+ ProviderOptionsParser& AddAssignmentToEnumReference(+ const std::string& name, const EnumNameMapping<EnumType>& mapping, EnumType& dest) {+ return AddValueParser(+ name,+ [&mapping, &dest](const std::string& value_str) -> Status {+ return NameToEnum(mapping, value_str, dest);+ });+ }++ /**+ * Parses the given provider options.+ */+ Status Parse(const ProviderOptions& options) const {+ for (const auto& option : options) {+ const auto& name = option.first;+ const auto& value_str = option.second;++ const auto value_parser_it = value_parsers_.find(name);+ ORT_RETURN_IF(+ value_parser_it == value_parsers_.end(),+ "Unknown provider option: \"", name, "\".");++ const auto parse_status = value_parser_it->second(value_str);+ ORT_RETURN_IF_NOT(+ parse_status.IsOK(),+ "Failed to parse provider option \"", name, "\": ", parse_status.ErrorMessage());+ }++ return Status::OK();+ }++ private:+ using ValueParser = std::function<Status(const std::string&)>;+ std::unordered_map<std::string, ValueParser> value_parsers_;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/provider_shutdown.h b/include/onnxruntime/core/framework/provider_shutdown.hnew file mode 100644index 0000000..7488f12--- /dev/null+++ b/include/onnxruntime/core/framework/provider_shutdown.h@@ -0,0 +1,8 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {+void UnloadSharedProviders();+}diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.hnew file mode 100644index 0000000..85f6e2f--- /dev/null+++ b/include/onnxruntime/core/framework/run_options.h@@ -0,0 +1,40 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <string>+#include <atomic>+#include "core/session/onnxruntime_c_api.h"++/**+ * Configuration information for a Run call.+ */+struct OrtRunOptions {+ /// Log severity. See https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h+ /// Default = -1 (use the log severity from the InferenceSession that the Run is for).+ int run_log_severity_level = -1;+ int run_log_verbosity_level = 0; ///< VLOG level if debug build and run_log_severity_level is 0 (VERBOSE).+ std::string run_tag; ///< A tag for the Run() calls using this.++ // Set to 'true' to ensure the termination of all the outstanding Run() calls+ // that use this OrtRunOptions instance. Some of the outstanding Run() calls may+ // be forced to terminate with an error status.+ bool terminate = false;++ // Set to 'true' to run only the nodes from feeds to required fetches.+ // So it is possible that only some of the nodes are executed.+ bool only_execute_path_to_fetches = false;++#ifdef ENABLE_TRAINING+ // Set to 'true' to run in training mode.+ bool training_mode = true;+#endif++ OrtRunOptions() = default;+ ~OrtRunOptions() = default;+};++namespace onnxruntime {+using RunOptions = OrtRunOptions;+}diff --git a/include/onnxruntime/core/framework/sparse_tensor.h b/include/onnxruntime/core/framework/sparse_tensor.hnew file mode 100644index 0000000..ec0abb9--- /dev/null+++ b/include/onnxruntime/core/framework/sparse_tensor.h@@ -0,0 +1,72 @@+#pragma once++// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "core/framework/data_types.h"+#include "core/framework/tensor_shape.h"+#include "core/framework/tensor.h"++namespace onnxruntime {++/**+ * @brief This class implements SparseTensor.+ * We represent a SparseTensor as a triple <values, indices, shape>. "values" and "indices" themselves+ * are implemented as Tensors.+ * We follow the Tensor design for memory ownership/management: a sparse-tensor does not own the "value"+ * or "indices" tensors.+ */++class SparseTensor final {+ public:+ SparseTensor(MLDataType elt_type,+ const TensorShape& shape,+ size_t nnz,+ void* values_data,+ void* indices_data,+ const OrtMemoryInfo& memory_info);++ SparseTensor(MLDataType elt_type,+ const TensorShape& shape,+ size_t nnz,+ std::shared_ptr<IAllocator> allocator);++ ~SparseTensor() = default;++ // For now, disallow all copy, assignment, and move.+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SparseTensor);++ // Returns the number of entries in the values tensor (aka "NNZ" or "number of nonzero values")+ size_t NumValues() const { return static_cast<size_t>(values_.Shape().Size()); }++ const Tensor& Values() const {+ return values_;+ }++ const Tensor& Indices() const {+ return indices_;+ }++ const TensorShape& Shape() const {+ return shape_;+ }++ Tensor& MutableValues() {+ return values_;+ }++ Tensor& MutableIndices() {+ return indices_;+ }++ //TensorShape& MutableShape() {+ // return shape_;+ //}++ private:+ Tensor values_;+ Tensor indices_;+ TensorShape shape_; // The shape of corresponding dense-tensor.+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/tensor.h b/include/onnxruntime/core/framework/tensor.hnew file mode 100644index 0000000..5c67796--- /dev/null+++ b/include/onnxruntime/core/framework/tensor.h@@ -0,0 +1,246 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <stddef.h>+#include <iostream>+#include <string>+#include <vector>++#include "gsl/gsl"+#include "core/common/common.h"+#include "core/framework/allocator.h"+#include "core/framework/tensor_shape.h"+#include "onnxruntime_config.h"+#include "core/framework/data_types.h"+#include "core/framework/data_types_internal.h"++namespace onnxruntime {+// TODO: Do we need this class or is IAllocator::MakeUniquePtr sufficient/better+class BufferDeleter {+ public:+ BufferDeleter() : alloc_(nullptr) {}+ BufferDeleter(AllocatorPtr alloc)+ : alloc_(alloc) {}++ void operator()(void* p) const {+ if (alloc_)+ alloc_->Free(p);+ }++ private:+ // TODO: we may need consider the lifetime of alloc carefully+ // The alloc_ here is the allocator that used to allocate the buffer+ // And need go with the unique_ptr together. If it is using our internal+ // allocator, it is ok as our allocators are global managed. But if it+ // is provide by user, user need to be very careful about it.+ // A weak_ptr may be a choice to reduce the impact, but that require to+ // change our current allocator mgr to use shared_ptr. Will revisit it+ // later.+ AllocatorPtr alloc_;+};++using BufferUniquePtr = std::unique_ptr<void, BufferDeleter>;+using BufferNakedPtr = void*;+//TODO:ensure dtype_!=nullptr+#ifdef __GNUC__+#pragma GCC diagnostic push+#ifdef HAS_NULL_DEREFERENCE+#pragma GCC diagnostic ignored "-Wnull-dereference"+#endif+#endif+/*+ We want to keep tensor as simple as possible, it is just a placeholder+ for a piece of memory, with additional shape information.+ Memory is owned and managed by Executor / Workspace, so Tensor just uses+ it, and won't do any allocation / release.+*/+class Tensor final {+ public:+ Tensor() = default; // to allow creating vector<Tensor> to support seq(tensor)++ /**+ * Create tensor with given type, shape, pre-allocated memory and allocator info.+ * This function won't check if the preallocated buffer(p_data) has enough room for the shape.+ * \param data A preallocated buffer. Can be NULL if the shape is empty.+ * Tensor does not own the data and will not delete it+ * \param alloc Where the buffer('data') was allocated from+ * \param offset Offset in bytes to start of Tensor within p_data.+ */+ Tensor(MLDataType p_type, const TensorShape& shape, void* p_data, const OrtMemoryInfo& alloc,+ ptrdiff_t offset = 0);++ /**+ * Deprecated. The orginal design is this Tensor class won't do any allocation / release.+ * However, this function will allocate the buffer for the shape, and do placement new if p_type is string tensor.+ */+ Tensor(MLDataType p_type, const TensorShape& shape, std::shared_ptr<IAllocator> allocator);++ ~Tensor();++ //Move is allowed+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(Tensor);++ Tensor(Tensor&& other) noexcept;++ Tensor& operator=(Tensor&& other) noexcept;++ /**+ Returns the data type.+ */+ MLDataType DataType() const { return dtype_; }++ /**+ Returns the data type enum constant+ @remarks Use utils::ToTensorProtoElementType<T> for comparison.+ */+ int32_t GetElementType() const {+ return dtype_->GetDataType();+ }++ // Check if contains string data. This is a separate+ // interface bc it is frequently used.+ bool IsDataTypeString() const {+ return utils::IsPrimitiveDataType<std::string>(dtype_);+ }++ // Checks if the Tensor contains data type T+ template <class T>+ bool IsDataType() const {+ return utils::IsPrimitiveDataType<T>(dtype_);+ }++ /**+ Returns the shape of the tensor.+ */+ const TensorShape& Shape() const noexcept { return shape_; }++ /**+ Returns the location of the tensor's memory+ */+ const OrtMemoryInfo& Location() const { return alloc_info_; }++ /**+ May return nullptr if tensor size is zero+ */+ template <typename T>+ T* MutableData() {+ // Type check+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",+ "T ", "!=", dtype_);+ return reinterpret_cast<T*>(static_cast<char*>(p_data_) + byte_offset_);+ }++ /**+ May return nullptr if tensor size is zero+ */+ template <typename T>+ gsl::span<T> MutableDataAsSpan() {+ // Type check+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",+ "T ", "!=", dtype_);+ T* data = reinterpret_cast<T*>(static_cast<char*>(p_data_) + byte_offset_);+ return gsl::make_span(data, static_cast<size_t>(shape_.Size()));+ }++ template <typename T>+ const T* Data() const {+ // Type check+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",+ "T ", "!=", dtype_);+ return reinterpret_cast<const T*>(static_cast<char*>(p_data_) + byte_offset_);+ }++ template <typename T>+ gsl::span<const T> DataAsSpan() const {+ // Type check+ ORT_ENFORCE(utils::IsPrimitiveDataType<T>(dtype_), "Tensor type mismatch. ",+ "T ", "!=", dtype_);+ const T* data = reinterpret_cast<const T*>(static_cast<char*>(p_data_) + byte_offset_);+ return gsl::make_span(data, static_cast<typename gsl::span<T>::index_type>(shape_.Size()));+ }++ void* MutableDataRaw(MLDataType type) {+ ORT_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_);+ return static_cast<char*>(p_data_) + byte_offset_;+ }++ const void* DataRaw(MLDataType type) const {+ ORT_ENFORCE(type == dtype_, "Tensor type mismatch.", type, "!=", dtype_);+ return static_cast<char*>(p_data_) + byte_offset_;+ }++ void* MutableDataRaw() noexcept {+ return static_cast<char*>(p_data_) + byte_offset_;+ }++ const void* DataRaw() const noexcept {+ return static_cast<char*>(p_data_) + byte_offset_;+ }++ bool OwnsBuffer() const noexcept {+ return buffer_deleter_ != nullptr;+ }++ /**+ * Resizes the tensor without touching underlying storage.+ * This requires the total size of the tensor to remains constant.+ * @warning this function is NOT thread-safe.+ */+ inline void Reshape(const TensorShape& new_shape) {+ ORT_ENFORCE(shape_.Size() == new_shape.Size(),+ "Tensor size (" + std::to_string(shape_.Size()) ++ ") != new size (" + std::to_string(new_shape.Size()) + ")");+ shape_ = new_shape;+ }++ /**+ * Get the byte offset with respect to the p_data+ * @warning this is a temporary solution for reusing the buffer bigger than needed.+ * @warning use with caution - make sure you do boundary check before calling this method (see view.cc)+ */+ inline ptrdiff_t ByteOffset() const {+ return byte_offset_;+ }++ /**+ * Set the byte offset with respect to the p_data+ * @warning this is a temporary solution for reusing the buffer bigger than needed.+ */+ inline void SetByteOffset(ptrdiff_t byte_offset) {+ byte_offset_ = byte_offset;+ }++ /**+ The number of bytes of data.+ */+ size_t SizeInBytes() const;++ // More API methods.+ private:+ void Init(MLDataType p_type,+ const TensorShape& shape,+ void* p_raw_data,+ AllocatorPtr deleter,+ ptrdiff_t offset = 0);++ void ReleaseBuffer();++ void* p_data_;+ /**+ if buffer_deleter_ is null, it means tensor does not own the buffer.+ otherwise tensor will use the deleter to release the buffer when+ tensor is released.+ */+ AllocatorPtr buffer_deleter_;++ TensorShape shape_;+ const PrimitiveDataTypeBase* dtype_;+ OrtMemoryInfo alloc_info_;+ ptrdiff_t byte_offset_;+};+#ifdef __GNUC__+#pragma GCC diagnostic pop+#endif+} // namespace onnxruntimediff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.hnew file mode 100644index 0000000..9a2609b--- /dev/null+++ b/include/onnxruntime/core/framework/tensor_shape.h@@ -0,0 +1,143 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#include <iosfwd>+#include <vector>+#include <algorithm>+#include <string>+#include <cstring>+#include "onnxruntime_config.h"++namespace onnxruntime {+#ifdef __GNUC__+#pragma GCC diagnostic push+#ifdef HAS_NULL_DEREFERENCE+#pragma GCC diagnostic ignored "-Wnull-dereference"+#endif+#endif+class TensorShape : private std::vector<int64_t> {+ // TODO - Use a custom STL allocator to avoid heap allocations in the common case.+ // We use negative numbers for unknown symbolic dimension. Each negative+ // number represents a unique symbolic dimension.+ // Private inheritance is used to prevent ambiguity of element versus dimension size+ public:+ TensorShape() = default;++ TensorShape(const TensorShape& /*other*/) = default;+ TensorShape& operator=(const TensorShape& /*other*/) = default;++ TensorShape(TensorShape&& /*other*/) = default;+ TensorShape& operator=(TensorShape&& /*other*/) = default;++ TensorShape(const std::vector<int64_t>& dims) : std::vector<int64_t>(dims) {}++ TensorShape(std::vector<int64_t>&& dims) : std::vector<int64_t>(std::move(dims)) {}++ TensorShape(const std::initializer_list<int64_t>& dims) : std::vector<int64_t>(dims) {}++ TensorShape(const int64_t* dimension_sizes, size_t dimension_count);++ TensorShape(const std::vector<int64_t>& dims, size_t start, size_t end);++ /**+ Return the dimension specified by <idx>.+ */+ const int64_t& operator[](size_t idx) const {+ return std::vector<int64_t>::operator[](static_cast<int>(idx));+ }++ int64_t& operator[](size_t idx) {+ return std::vector<int64_t>::operator[](static_cast<int>(idx));+ }++ bool operator==(const TensorShape& other) const noexcept {+ auto thisVector = static_cast<const std::vector<int64_t>*>(this);+ auto otherVector = static_cast<const std::vector<int64_t>*>(&other);+ return *thisVector == *otherVector;+ }++ bool operator!=(const TensorShape& other) const noexcept {+ return !(*this == other);+ }++ size_t NumDimensions() const noexcept {+ return size();+ }++ /**+ Copy dims into an array with given size+ */+ void CopyDims(int64_t* dims, size_t num_dims) const {+ memcpy(dims, data(), sizeof(value_type) * std::min(num_dims, NumDimensions()));+ }++ /**+ Return underlying vector representation.+ */+ const std::vector<int64_t>& GetDims() const { return *this; }++ /**+ * Return the total number of elements. Returns 1 for an empty (rank 0) TensorShape.+ *+ * May return -1+ */+ int64_t Size() const;++ /**+ Return the total number of elements up to the specified dimension.+ If the dimension interval is empty (dimension == 0), return 1.+ @param dimension Return size up to this dimension. Value must be between 0 and this->NumDimensions(), inclusive.+ */+ int64_t SizeToDimension(size_t dimension) const;++ /**+ Return the total number of elements from the specified dimension to the end of the tensor shape.+ If the dimension interval is empty (dimension == this->NumDimensions()), return 1.+ @param dimension Return size from this dimension to the end. Value must be between 0 and this->NumDimensions(),+ inclusive.+ */+ int64_t SizeFromDimension(size_t dimension) const;++ /**+ Return a new TensorShape of the dimensions from dimstart to dimend.+ */+ TensorShape Slice(size_t dimstart, size_t dimend) const;++ /**+ Return a new TensorShape of the dimensions from dimstart to end.+ */+ TensorShape Slice(size_t dimstart) const;++ /**+ output dimensions nicely formatted+ */+ std::string ToString() const;++ /**+ Calculate size between start and end.+ Assumes start and end are between 0 and this->NumDimensions(), inclusive, and that+ start < end.+ */+ int64_t SizeHelper(size_t start, size_t end) const;++ /**+ empty shape or 1D shape (1) is regarded as scalar tensor+ */+ bool IsScalar() const {+ size_t len = size();+ return len == 0 || (len == 1 && operator[](0) == 1);+ }++ static const TensorShape& ReinterpretBaseType(const std::vector<int64_t>& dimensions) {+ static_assert(sizeof(TensorShape) == sizeof(std::vector<int64_t>), "Size of TensorShape prevents safe casting from vector");+ return *static_cast<const TensorShape*>(&dimensions);+ }+};+#ifdef __GNUC__+#pragma GCC diagnostic pop+#endif+// operator<< to nicely output to a stream+std::ostream& operator<<(std::ostream& out, const TensorShape& shape);++} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/basic_types.h b/include/onnxruntime/core/graph/basic_types.hnew file mode 100644index 0000000..7c1203f--- /dev/null+++ b/include/onnxruntime/core/graph/basic_types.h@@ -0,0 +1,46 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include <unordered_map>+#include <unordered_set>+#include <string>+#include <cstdint>+#include <memory>+#include <functional>++namespace ONNX_NAMESPACE {+class ValueInfoProto;+class TensorProto;+class SparseTensorProto;+class TypeProto;+class AttributeProto;+// define types that would come from the ONNX library if we were building against it.+#if defined(ORT_MINIMAL_BUILD)+using OperatorSetVersion = int;+#endif++} // namespace ONNX_NAMESPACE++namespace onnxruntime {+using NodeIndex = size_t;+using Version = int64_t;+using NodeArgInfo = ONNX_NAMESPACE::ValueInfoProto;+using InitializedTensorSet = std::unordered_map<std::string, const ONNX_NAMESPACE::TensorProto*>;+using ArgNameToTypeMap = std::unordered_map<std::string, ONNX_NAMESPACE::TypeProto>;+using ProviderType = const std::string&;+// TODO - Evaluate switching the types below to support transparent comparators and enable+// lookups based on gsl::cstring_span<> and std::string_view. This would reduces allocations+// converting to std::string, but requires conversion to std::map<std::string, foo, std::less<>>+// instead of std::unordered_map<std::string, foo, [std::less<foo>]>.++using NodeAttributes = std::unordered_map<std::string, ONNX_NAMESPACE::AttributeProto>;+class IOnnxRuntimeOpSchemaCollection;+using IOnnxRuntimeOpSchemaCollectionPtr = std::shared_ptr<IOnnxRuntimeOpSchemaCollection>;+} // namespace onnxruntime++namespace onnxruntime {+class OpKernel;+class OpKernelInfo;++using KernelCreateFn = std::function<OpKernel*(const OpKernelInfo& info)>;+} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.hnew file mode 100644index 0000000..f723a39--- /dev/null+++ b/include/onnxruntime/core/graph/constants.h@@ -0,0 +1,38 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {++constexpr const char* kNoOp = "NoOp";+constexpr const char* kConstant = "Constant";+constexpr const char* kFunctionOp = "_kFunctionOp";+constexpr const char* kConstantValue = "value";+constexpr const char* kOnnxDomain = "";+constexpr const char* kOnnxDomainAlias = "ai.onnx";+constexpr const char* kMLDomain = "ai.onnx.ml";+constexpr const char* kMSDomain = "com.microsoft";+constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc";+constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers";+constexpr const char* kMSDmlDomain = "com.microsoft.dml";+constexpr const char* kNGraphDomain = "com.intel.ai";+constexpr const char* kMIGraphXDomain = "";+constexpr const char* kVitisAIDomain = "com.xilinx";++constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider";+constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider";+constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider";+constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider";+constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider";+constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider";+constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider";+constexpr const char* kNnapiExecutionProvider = "NnapiExecutionProvider";+constexpr const char* kRknpuExecutionProvider = "RknpuExecutionProvider";+constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider";+constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider";+constexpr const char* kAclExecutionProvider = "ACLExecutionProvider";+constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider";+constexpr const char* kRocmExecutionProvider = "ROCMExecutionProvider";++} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/function.h b/include/onnxruntime/core/graph/function.hnew file mode 100644index 0000000..cf9eefd--- /dev/null+++ b/include/onnxruntime/core/graph/function.h@@ -0,0 +1,41 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/common.h"+#include "core/graph/indexed_sub_graph.h"++namespace onnxruntime {+class Graph;+class Node;+} // namespace onnxruntime++namespace onnxruntime {++/**+@class Function+Class representing a Function.+*/+class Function {+ public:+ virtual ~Function() = default;++#if !defined(ORT_MINIMAL_BUILD)+ /** Gets the OpSchema for the Function. */+ virtual const ONNX_NAMESPACE::OpSchema& OpSchema() const = 0;+#endif++ /** Gets the Graph instance for the Function body subgraph. */+ virtual const onnxruntime::Graph& Body() const = 0;+};++/**+Create a new Function instance.+@param graph The graph containing the Function.+@param nodes_to_fuse the IndexedSubGraph to use for the Function.+*/+std::unique_ptr<Function> MakeFunction(const onnxruntime::Graph& graph,+ const IndexedSubGraph& nodes_to_fuse,+ const logging::Logger& logger);+} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.hnew file mode 100644index 0000000..d1841ff--- /dev/null+++ b/include/onnxruntime/core/graph/graph.h@@ -0,0 +1,1407 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <limits>+#include <memory>+#include <string>+#include <type_traits>+#include <unordered_map>+#include <unordered_set>++#include "core/common/common.h"+#include "core/common/const_pointer_container.h"+#include "core/common/path.h"+#include "core/common/status.h"+#include "core/common/logging/logging.h"+#include "core/graph/basic_types.h"+#include "core/graph/constants.h"+#include "core/graph/graph_nodes.h"+#include "core/graph/node_arg.h"+#include "core/graph/onnx_protobuf.h"+#include "core/graph/function.h"+#include "gsl/gsl"++namespace flatbuffers {+class FlatBufferBuilder;+template <typename T>+struct Offset;+} // namespace flatbuffers++namespace onnxruntime {+class Graph;+struct IndexedSubGraph;+class Model;+class OpSignature;++namespace experimental {+namespace fbs {+struct Graph;+struct Node;+struct NodeEdge;+} // namespace fbs+} // namespace experimental++/**+@class Node+Class representing a node in the graph.+*/+class Node {+ public:+ /** Node types */+ enum class Type {+ Primitive = 0, ///< The node refers to a primitive operator.+ Fused = 1, ///< The node refers to a function.+ };++ ~Node() = default;++ /**+ @class EdgeEnd+ Class representing the end of an edge. It could be an input or output edge end of a node.+ For the node's input edge end, it's the source end, as the destination end is the node itself.+ For the node's output edge end, it's the destination end, as the source end is the node itself.+ */+ class EdgeEnd {+ public:+ /**+ Construct an EdgeEnd+ @param node The source node if this is an input edge to the current node,+ or the destination node if this is an output edge from the current node.+ @param src_arg_index The node arg index of source node of the edge.+ @param dst_arg_index The node arg index of destination node of the edge.+ */+ EdgeEnd(const Node& node, int src_arg_index, int dst_arg_index) noexcept;++ /** Construct a control edge.+ @param node The node the edge joins to the current node.+ */+ explicit EdgeEnd(const Node& node) noexcept;++ /** Gets the Node that this EdgeEnd refers to. */+ const Node& GetNode() const noexcept { return *node_; }++ /** Gets the source arg index.+ @returns the source arg index of <*this> edge.*/+ int GetSrcArgIndex() const { return src_arg_index_; }++ /** Gets the destination arg index.+ @returns the destination arg index of <*this> edge.*/+ int GetDstArgIndex() const { return dst_arg_index_; }++ private:+ const Node* node_;+ const int src_arg_index_;+ const int dst_arg_index_;+ };++ /** Gets the Node's NodeIndex. */+ NodeIndex Index() const noexcept { return index_; }++ /** Gets the Node's name. */+ const std::string& Name() const noexcept { return name_; }++ /** Gets the Node's operator type. */+ const std::string& OpType() const noexcept { return op_type_; }++ /** Gets the domain of the OperatorSet that specifies the operator returned by #OpType. */+ const std::string& Domain() const noexcept { return domain_; }++ /** Gets the path of the owning model if any. */+ const Path& ModelPath() const noexcept;++ /** Gets the Node's execution priority.+ @remarks Lower value means higher priority */+ int Priority() const noexcept { return priority_; };++ /** Sets the execution priority of a node.+ @remarks Lower value means higher priority */+ void SetPriority(int priority) noexcept;++ /** Gets the node description. */+ const std::string& Description() const noexcept { return description_; }++ /** Gets the Node's Node::Type. */+ Node::Type NodeType() const noexcept { return node_type_; }++ /** Gets the opset version that the Node's operator was first defined in.+ @returns Opset version. If -1 the Node's operator has not been set.+ @remarks Prefer over Op()->SinceVersion() as Op() is disabled in a minimal build+ */+ int SinceVersion() const noexcept { return since_version_; }++#if !defined(ORT_MINIMAL_BUILD)+ /** Gets the Node's OpSchema.+ @remarks The graph containing this node must be resolved, otherwise nullptr will be returned. */+ const ONNX_NAMESPACE::OpSchema* Op() const noexcept { return op_; }++ /**+ Gets the function body if applicable otherwise nullptr+ @param try_init_func_body If not already intialized, initialize the function body+ (only applicable to operators which are defined as function in ONNX spec).+ Function body can be initialized in 2 cases :+ 1. For nodes of type "Fused"+ 2. For nodes which are defined as functions in ONNX spec (example: DynamicQuantizeLinear)+ For all other cases this will always return nullptr.+ Nodes of type "Fused" are created during partitioning and the function body+ initialization for such nodes also happens during node creation. Therefore,+ initialization of function body will happen via this method only in case 2 mentioned above.+ */+ const Function* GetFunctionBody(bool try_init_func_body = true);++ /** Gets the function body if applicable otherwise nullptr. */+ const Function* GetFunctionBody() const noexcept { return func_body_; }++#endif++ /**+ Helper to iterate through the container returned by #InputDefs() or #OutputDefs() and call the provided function.+ @param node_args Collection of NodeArgs returned by #InputDefs() or #OutputDefs()+ @param func Function to call for each valid NodeArg in the node_args. The function is called with the NodeArg+ and the index number in the container.+ @returns common::Status with success or error information.+ @remarks Returns immediately on error.+ */+ static common::Status ForEachWithIndex(const ConstPointerContainer<std::vector<NodeArg*>>& node_args,+ std::function<common::Status(const NodeArg& arg, size_t index)> func) {+ for (size_t index = 0; index < node_args.size(); ++index) {+ auto arg = node_args[index];+ if (!arg->Exists())+ continue;+ ORT_RETURN_IF_ERROR(func(*arg, index));+ }+ return common::Status::OK();+ }++ /** Gets the count of arguments for each of the Node's explicit inputs. */+ const std::vector<int>& InputArgCount() const noexcept { return definitions_.input_arg_count; }++ /** Gets the Node's input definitions.+ @remarks requires ConstPointerContainer wrapper to apply const to the NodeArg pointers so access is read-only. */+ ConstPointerContainer<std::vector<NodeArg*>> InputDefs() const noexcept {+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.input_defs);+ }++ /** Gets the implicit inputs to this Node.+ If this Node contains a subgraph, these are the NodeArg's that are implicitly consumed by Nodes within that+ subgraph. e.g. If and Loop operators.*/+ ConstPointerContainer<std::vector<NodeArg*>> ImplicitInputDefs() const noexcept {+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.implicit_input_defs);+ }++ /** Gets the Node's output definitions.+ @remarks requires ConstPointerContainer wrapper to apply const to the NodeArg pointers so access is read-only. */+ ConstPointerContainer<std::vector<NodeArg*>> OutputDefs() const noexcept {+ return ConstPointerContainer<std::vector<NodeArg*>>(definitions_.output_defs);+ }++#if !defined(ORT_MINIMAL_BUILD)+ /**+ Helper to iterate through the container returned by #MutableInputDefs() or #MutableOutputDefs() and call the provided function.+ @param node_args Collection of NodeArgs returned by #MutableInputDefs() or #MutableOutputDefs()+ @param func Function to call for each valid NodeArg in the node_args. The function is called with the NodeArg+ and the index number in the container.+ @returns common::Status with success or error information.+ @remarks Returns immediately on error.+ */+ static common::Status ForEachMutableWithIndex(std::vector<NodeArg*>& node_args,+ std::function<common::Status(NodeArg& arg, size_t index)> func) {+ for (size_t index = 0; index < node_args.size(); ++index) {+ auto arg = node_args[index];+ if (!arg->Exists())+ continue;+ ORT_RETURN_IF_ERROR(func(*arg, index));+ }+ return common::Status::OK();+ }+ /** Gets a modifiable count of arguments for each of the Node's explicit inputs.+ @todo This should be removed in favor of a method that updates the input args and the count.+ Currently these operations are separate which is not a good setup. */+ std::vector<int>& MutableInputArgsCount() { return definitions_.input_arg_count; }++ /** Gets a modifiable collection of the Node's input definitions. */+ std::vector<NodeArg*>& MutableInputDefs() noexcept {+ return definitions_.input_defs;+ }++ /** Gets a modifiable collection of the Node's implicit input definitions. */+ std::vector<NodeArg*>& MutableImplicitInputDefs() noexcept {+ return definitions_.implicit_input_defs;+ }++ /** Gets a modifiable collection of the Node's output definitions. */+ std::vector<NodeArg*>& MutableOutputDefs() noexcept {+ return definitions_.output_defs;+ }+#endif // !defined(ORT_MINIMAL_BUILD)++ /** Struct to provide sorting between EdgeEnd instances based on NodeIndex first, and NodeArg::Name second. */+ struct EdgeEndCompare {+ bool operator()(const EdgeEnd& lhs, const EdgeEnd& rhs) const {+ if (lhs.GetNode().Index() == rhs.GetNode().Index()) {+ if (lhs.GetSrcArgIndex() == rhs.GetSrcArgIndex()) {+ return lhs.GetDstArgIndex() < rhs.GetDstArgIndex();+ }+ return lhs.GetSrcArgIndex() < rhs.GetSrcArgIndex();+ }+ return lhs.GetNode().Index() < rhs.GetNode().Index();+ }+ };++ using EdgeSet = std::set<EdgeEnd, EdgeEndCompare>;+ using EdgeConstIterator = EdgeSet::const_iterator;++ /**+ @class NodeConstIterator+ Class to provide const access to Node instances iterated via an EdgeConstIterator. */+ class NodeConstIterator {+ public:+ NodeConstIterator(EdgeConstIterator p_iter);++ bool operator==(const NodeConstIterator& p_other) const;++ bool operator!=(const NodeConstIterator& p_other) const;++ void operator++();+ void operator--();++ const Node& operator*() const;+ const Node* operator->() const;++ private:+ EdgeConstIterator m_iter;+ };++ // Functions defined to traverse a Graph as below.++ /** Gets an iterator to the beginning of the input nodes to this Node. */+ NodeConstIterator InputNodesBegin() const noexcept { return NodeConstIterator(relationships_.input_edges.cbegin()); };+ /** Gets an iterator to the end of the input nodes to this Node. */+ NodeConstIterator InputNodesEnd() const noexcept { return NodeConstIterator(relationships_.input_edges.cend()); }++ /** Gets an iterator to the beginning of the output nodes from this Node. */+ NodeConstIterator OutputNodesBegin() const noexcept {+ return NodeConstIterator(relationships_.output_edges.cbegin());+ }++ /** Gets an iterator to the end of the output nodes from this Node. */+ NodeConstIterator OutputNodesEnd() const noexcept { return NodeConstIterator(relationships_.output_edges.cend()); }++ /** Gets an iterator to the beginning of the input edges to this Node.+ @remarks There are no nullptr entries in this collection. */+ EdgeConstIterator InputEdgesBegin() const noexcept { return relationships_.input_edges.cbegin(); }++ /** Gets an iterator to the end of the input edges to this Node. */+ EdgeConstIterator InputEdgesEnd() const noexcept { return relationships_.input_edges.cend(); }++ /** Gets an iterator to the beginning of the output edges from this Node.+ @remarks There are no nullptr entries in this collection. */+ EdgeConstIterator OutputEdgesBegin() const noexcept { return relationships_.output_edges.cbegin(); }++ /** Gets an iterator to the end of the output edges from this Node. */+ EdgeConstIterator OutputEdgesEnd() const noexcept { return relationships_.output_edges.cend(); }++ /** Gets the Node's control inputs. */+ const std::set<std::string>& ControlInputs() const noexcept { return relationships_.control_inputs; }++ /** Gets the number of input edges to this Node */+ size_t GetInputEdgesCount() const noexcept { return relationships_.input_edges.size(); }++ /** Gets the number of output edges from this Node */+ size_t GetOutputEdgesCount() const noexcept { return relationships_.output_edges.size(); }++ /** Add an attribute to this Node with specified attribute name and value. */+ void AddAttribute(const std::string& attr_name, const ONNX_NAMESPACE::AttributeProto& value);++#define ADD_ATTR_INTERFACES(TypeName) \+ void AddAttribute(const std::string& attr_name, const TypeName& value); \+ void AddAttribute(const std::string& attr_name, \+ const std::vector<TypeName>& values);++ ADD_ATTR_INTERFACES(int64_t)+ ADD_ATTR_INTERFACES(float)+ ADD_ATTR_INTERFACES(std::string)+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::TensorProto)+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::GraphProto)+ ADD_ATTR_INTERFACES(ONNX_NAMESPACE::SparseTensorProto)++ /** Gets the Node's attributes. */+ const NodeAttributes& GetAttributes() const noexcept { return attributes_; }++#if !defined(ORT_MINIMAL_BUILD)+ /** Remove the specified attribute from this Node */+ bool ClearAttribute(const std::string& attr_name);++ /** Gets the Node's mutable attributes. */+ NodeAttributes& GetMutableAttributes() noexcept { return attributes_; }++ /** Gets the Graph instance that is instantiated from a GraphProto attribute during Graph::Resolve.+ @param attr_name Attribute name for the GraphProto attribute.+ @returns nullptr if the Graph instance has not been instantiated or attribute does not contain a GraphProto.+ */+ const Graph* GetGraphAttribute(const std::string& attr_name) const;++ /** Gets the mutable Graph instance that is instantiated from a GraphProto attribute during Graph::Resolve.+ @param attr_name Attribute name for the GraphProto attribute.+ @returns nullptr if the Graph instance has not been instantiated or attribute does not contain a GraphProto.+ */+ Graph* GetMutableGraphAttribute(const std::string& attr_name);+#endif // !defined(ORT_MINIMAL_BUILD)++ /** Checks if the Node contains at least one subgraph (this is the case for control flow operators, such as If, Scan, Loop).+ @returns true if the Node contains a subgraph.+ */+ bool ContainsSubgraph() const {+ return !attr_to_subgraph_map_.empty();+ }++ /** Get the const subgraphs from a node.+ @remarks Creates a new vector so calling ContainsSubgraphs first is preferred. */+ std::vector<gsl::not_null<const Graph*>> GetSubgraphs() const;++ /** Gets a map of attribute name to the mutable Graph instances for all subgraphs of the Node.+ @returns Map of the attribute name that defines the subgraph to the subgraph's Graph instance.+ nullptr if the Node has no subgraphs.+ */+ const std::unordered_map<std::string, gsl::not_null<Graph*>>& GetAttributeNameToMutableSubgraphMap() {+ return attr_to_subgraph_map_;+ }++ /** Gets the execution ProviderType that this node will be executed by. */+ ProviderType GetExecutionProviderType() const noexcept { return execution_provider_type_; }++ /** Sets the execution ProviderType that this Node will be executed by. */+ void SetExecutionProviderType(ProviderType execution_provider_type) {+ execution_provider_type_ = execution_provider_type;+ }++ /** Call the provided function for all explicit inputs, implicit inputs, and outputs of this Node.+ If the NodeArg is an explicit or implicit input, is_input will be true when func is called.+ @param include_missing_optional_defs Include NodeArgs that are optional and were not provided+ i.e. NodeArg::Exists() == false.+ */+ void ForEachDef(std::function<void(const onnxruntime::NodeArg&, bool is_input)> func,+ bool include_missing_optional_defs = false) const;++#if !defined(ORT_MINIMAL_BUILD)+ /** Replaces any matching definitions in the Node's explicit inputs or explicit outputs.+ @param replacements Map of current NodeArg to replacement NodeArg.+ */+ void ReplaceDefs(const std::map<const onnxruntime::NodeArg*, onnxruntime::NodeArg*>& replacements);++ /** Gets the NodeProto representation of this Node.+ @param update_subgraphs Update the GraphProto values for any subgraphs in the returned NodeProto.+ If graph optimization has been run this is most likely required+ to ensure the complete Graph is valid.+ */+ void ToProto(ONNX_NAMESPACE::NodeProto& proto, bool update_subgraphs = false) const;++ Status SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,+ flatbuffers::Offset<onnxruntime::experimental::fbs::Node>& fbs_node) const;++ flatbuffers::Offset<onnxruntime::experimental::fbs::NodeEdge>+ SaveEdgesToOrtFormat(flatbuffers::FlatBufferBuilder& builder) const;++#endif++#if defined(ENABLE_ORT_FORMAT_LOAD)+ static Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Node& fbs_node, Graph& graph,+ const logging::Logger& logger, std::unique_ptr<Node>& node);++ Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Node& fbs_node, const logging::Logger& logger);+ Status LoadEdgesFromOrtFormat(const onnxruntime::experimental::fbs::NodeEdge& fbs_node_edgs, const Graph& graph);+#endif++ /**+ @class Definitions+ The input and output definitions for this Node.+ */+ class Definitions {+ public:+ Definitions() = default;++ /** The Node's explicit input definitions. */+ std::vector<NodeArg*> input_defs;++ /**+ The number of inputs for each argument of the operator or function which this node refers.+ @remarks For example, #input_defs has 10 elements (inputs), and #input_arg_count is {4, 6}.+ This means that 4 elements (inputs) of input_defs map to the first argument of the operator or function, and+ the other 6 map to the second argument.+ */+ std::vector<int> input_arg_count;++ /** The Node's output definitions. */+ std::vector<NodeArg*> output_defs;++ /** The Node's implicit input definitions if the Node contains one or more subgraphs+ (i.e. GraphProto attributes) and the subgraph/s implicitly consume these values.+ @remarks For example, a subgraph in an 'If' node gets all its input values via this mechanism rather than+ there being explicit inputs to the 'If' node that are passed to the subgraph.+ They are pseudo-inputs to this Node as it has an implicit dependency on them. */+ std::vector<NodeArg*> implicit_input_defs;++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Definitions);+ };++ /**+ @class Relationships+ Defines the relationships between this Node and other Nodes in the Graph.+ */+ class Relationships {+ public:+ Relationships() = default;++ void Clear() noexcept {+ input_edges.clear();+ output_edges.clear();+ control_inputs.clear();+ }++ /** The edges for Nodes that provide inputs to this Node. */+ EdgeSet input_edges;++ /** The edges for Nodes that receive outputs from this Node. */+ EdgeSet output_edges;++ /** The Node names of the control inputs to this Node. */+ std::set<std::string> control_inputs;++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Relationships);+ };++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Node);++ // NOTE: This friendship relationship should ONLY be used for calling methods of the Node class and not accessing+ // the data members directly, so that the Node can maintain its internal invariants.+ friend class Graph;++ Node(NodeIndex index, Graph& graph) : index_(index), graph_(&graph) {}++#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)+ void Init(const std::string& name,+ const std::string& op_type,+ const std::string& description,+ const std::vector<NodeArg*>& input_args,+ const std::vector<NodeArg*>& output_args,+ const NodeAttributes* attributes,+ const std::string& domain);++ // internal only method to allow selected classes to directly alter the input/output definitions and arg counts+ Definitions& MutableDefinitions() noexcept;++ // internal only method to allow selected classes to directly alter the links between nodes.+ Relationships& MutableRelationships() noexcept;++ void SetNodeType(Node::Type node_type) noexcept { node_type_ = node_type; }+#endif++ // create a Graph instance for an attribute that contains a GraphProto+ void CreateSubgraph(const std::string& attr_name);++ const std::vector<std::unique_ptr<Graph>>& MutableSubgraphs() noexcept { return subgraphs_; }++ // validate and update the input arg count+ common::Status UpdateInputArgCount();++ void SetFunctionBody(const Function& func);++ const Definitions& GetDefinitions() const noexcept { return definitions_; }+ const Relationships& GetRelationships() const noexcept { return relationships_; }++ // Node index. Default to impossible value rather than 0.+ NodeIndex index_ = std::numeric_limits<NodeIndex>::max();++ // Node name.+ std::string name_;++ // Node operator type.+ std::string op_type_;++ // OperatorSet domain of op_type_.+ std::string domain_;++#if !defined(ORT_MINIMAL_BUILD)+ // OperatorSchema that <*this> node refers to.+ const ONNX_NAMESPACE::OpSchema* op_ = nullptr;+#endif++ // Execution priority, lower value for higher priority+ int priority_ = 0;++ // set from op_->SinceVersion() or via deserialization when OpSchema is not available+ int since_version_ = -1;++ Node::Type node_type_ = Node::Type::Primitive;++ // The function body is owned by graph_+ const Function* func_body_ = nullptr;++ // Node doc string.+ std::string description_;++ // input/output defs and arg count+ Definitions definitions_;++ // Relationships between this node and others in the graph+ Relationships relationships_;++ // Device.+ std::string execution_provider_type_;++ // Map from attribute name to attribute.+ // This allows attribute adding and removing.+ NodeAttributes attributes_;++ // Graph that contains this Node+ Graph* graph_;++ // Map of attribute name to the Graph instance created from the GraphProto attribute+ std::unordered_map<std::string, gsl::not_null<Graph*>> attr_to_subgraph_map_;++ // Graph instances for subgraphs that are owned by this Node+ std::vector<std::unique_ptr<Graph>> subgraphs_;+};++/**+@class Graph+The Graph representation containing the graph inputs and outputs, the Node instances,+and the edges connecting the nodes.+*/+class Graph {+ public:+ /** Gets the Graph name. */+ const std::string& Name() const noexcept;++ /** Gets the Graph description. */+ const std::string& Description() const noexcept;++ /** Gets the path of the owning model, if any. */+ const Path& ModelPath() const;++ /** Returns true if this is a subgraph or false if it is a high-level graph. */+ bool IsSubgraph() const { return parent_graph_ != nullptr; }++ /** Returns the parent graph if this is a subgraph */+ const Graph* ParentGraph() const { return parent_graph_; }++ /** Returns the mutable parent graph if this is a subgraph */+ Graph* MutableParentGraph() { return parent_graph_; }++#if !defined(ORT_MINIMAL_BUILD)+ /** Sets the Graph name. */+ void SetName(const std::string& name);++ /** Gets the Graph description. */+ void SetDescription(const std::string& description);++ /** Add an initializer tensor to the Graph. */+ void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor_proto);++ /** Replaces the initializer tensor with the same name as the given initializer tensor.+ The replacement initializer tensor must have the same type and shape as the existing initializer tensor.++ Note: This currently has linear time complexity. There is room for improvement but it would likely require changes to+ how initializer tensors are stored and tracked.+ */+ common::Status ReplaceInitializedTensor(const ONNX_NAMESPACE::TensorProto& new_initializer);+#endif // !defined(ORT_MINIMAL_BUILD)++ /** Remove the initializer tensor with the provided name from the Graph. */+ void RemoveInitializedTensor(const std::string& tensor_name);++ /** Check if a given name is an initializer tensor's name in this graph. */+ bool IsInitializedTensor(const std::string& name) const;++ /** Gets an initializer tensor with the provided name.+ @param[out] value Set to the TensorProto* if the initializer is found, or nullptr if not.+ @returns True if found.+ */+ bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const;++ /** Gets all the initializer tensors in this Graph. */+ const InitializedTensorSet& GetAllInitializedTensors() const noexcept { return name_to_initial_tensor_; }++ /** Removes all initializer tensors from this Graph and releases the memory they were using. */+ void CleanAllInitializedTensors() noexcept;++ /** Returns true if an initializer value can be overridden by a graph input with the same name. */+ bool CanOverrideInitializer() const noexcept { return ir_version_ >= 4; }++ /** returns the initializer's TensorProto if 'name' is an initializer, is constant and+ cannot be overridden at runtime. If the initializer is not found or is not constant, a nullptr is returned.+ @param check_outer_scope If true and the graph is a subgraph,+ check ancestor graph/s for 'name' if not found in 'graph'.+ @remarks check_outer_scope of true is not supported in a minimal build+ */+ const ONNX_NAMESPACE::TensorProto* GetConstantInitializer(const std::string& name, bool check_outer_scope) const;++ /** Gets the Graph inputs excluding initializers.+ These are the required inputs to the Graph as the initializers can be optionally overridden via graph inputs.+ @remarks Contains no nullptr values. */+ const std::vector<const NodeArg*>& GetInputs() const noexcept { return graph_inputs_excluding_initializers_; }++ /** Gets the Graph inputs including initializers.+ This is the full set of inputs, in the same order as defined in the GraphProto.+ @remarks Contains no nullptr values. */+ const std::vector<const NodeArg*>& GetInputsIncludingInitializers() const noexcept {+ return graph_inputs_including_initializers_;+ }++ /** Return true if "node_arg" is a input or an initializer. Otherwise, returns false. */+ bool IsInputsIncludingInitializers(const NodeArg* node_arg) const noexcept {+ return std::find(graph_inputs_including_initializers_.begin(), graph_inputs_including_initializers_.end(), node_arg) != graph_inputs_including_initializers_.end();+ }++ /** Gets the Graph inputs that are initializers+ These are overridable initializers. This is a difference between+ graph_inputs_including_initializers_ and graph_inputs_excluding_initializers_+ @remarks Contains no nullptr values. */+ const std::vector<const NodeArg*>& GetOverridableInitializers() const {+ return graph_overridable_initializers_;+ }++ /** Gets the Graph outputs.+ @remarks Contains no nullptr values.*/+ const std::vector<const NodeArg*>& GetOutputs() const noexcept { return graph_outputs_; }++ bool IsOutput(const NodeArg* node_arg) const noexcept {+ return std::find(graph_outputs_.begin(), graph_outputs_.end(), node_arg) != graph_outputs_.end();+ }++ /** Returns a vector with the indexes of the outputs of the given Node that are also Graph outputs. */+ std::vector<int> GetNodeOutputsInGraphOutputs(const Node& node) const {+ int output_idx = 0;+ std::vector<int> indexes;+ for (auto output_def : node.OutputDefs()) {+ if (std::find(GetOutputs().cbegin(), GetOutputs().cend(), output_def) != GetOutputs().cend()) {+ indexes.push_back(output_idx);+ }++ ++output_idx;+ }++ return indexes;+ }++ /** Gets the NodeArgs that represent value_info instances in the Graph.+ These are the values that are neither Graph inputs nor outputs.+ @remarks Contains no nullptr values. */+ const std::vector<const NodeArg*>& GetValueInfo() const noexcept { return value_info_; }++#if !defined(ORT_MINIMAL_BUILD)+ void AddValueInfo(const NodeArg* new_value_info);+#endif++ /** Gets the Node with the specified node index.+ @returns Node instance if found. nullptr if node_index is invalid or node has been freed.+ */+ const Node* GetNode(NodeIndex node_index) const { return NodeAtIndexImpl(node_index); }++ /** Gets the mutable Node with the specified node index.+ @returns Mutable Node instance if found. nullptr if node_index is invalid or node has been freed.+ */+ Node* GetNode(NodeIndex node_index) { return NodeAtIndexImpl(node_index); }++ /** Get a GraphNodes instance that provides mutable access to all valid Nodes in the Graph. */+ GraphNodes& Nodes() noexcept { return iterable_nodes_; }++ /** Get a GraphNodes instance that provides const access to all valid Nodes in the Graph. */+ const GraphNodes& Nodes() const noexcept { return iterable_nodes_; }++ /** Get a ConstGraphNodes instance that provides access to a filtered set of valid Nodes in the Graph.+ @remarks We can't use GraphNodes as that would provide mutable access to the nodes by default, and we can't prevent+ that by returning a const instance of GraphNodes as we're creating a new instance here due to the filter+ being something we don't control (i.e. we have to return a new instance so it can't be const).+ */+ ConstGraphNodes FilteredNodes(GraphNodes::NodeFilterFunc&& filter_func) const noexcept {+ return ConstGraphNodes(nodes_, std::move(filter_func));+ }++ /** Gets the maximum NodeIndex value used in the Graph. */+ int MaxNodeIndex() const noexcept { return static_cast<int>(nodes_.size()); } //assume the casting won't overflow++ /** Gets the number of valid Nodes in the Graph.+ @remarks This may be smaller than MaxNodeIndex(), as Nodes may be removed during optimization.+ */+ int NumberOfNodes() const noexcept { return num_of_nodes_; }++ /** Gets the mutable NodeArg with the provided name.+ @returns Pointer to NodeArg if found, nullptr if not. */+ NodeArg* GetNodeArg(const std::string& name) {+ auto iter = node_args_.find(name);+ if (iter != node_args_.end()) {+ return iter->second.get();+ }+ return nullptr;+ }++ /** Gets the const NodeArg with the provided name.+ @returns Pointer to const NodeArg if found, nullptr if not. */+ const NodeArg* GetNodeArg(const std::string& name) const {+ return const_cast<Graph*>(this)->GetNodeArg(name);+ }++ // search this and up through any parent_graph_ instance for a NodeArg+ NodeArg* GetNodeArgIncludingParentGraphs(const std::string& node_arg_name);++ /** Gets a mutable NodeArg by name. Creates a new NodeArg that is owned by this Graph if not found.+ @param name The NodeArg name.+ @param[in] p_arg_type Optional TypeProto to use if the NodeArg needs to be created.+ @returns NodeArg reference.+ */+ NodeArg& GetOrCreateNodeArg(const std::string& name, const ONNX_NAMESPACE::TypeProto* p_arg_type) {+ auto iter = node_args_.find(name);+ if (iter != node_args_.end()) {+ return *(iter->second);+ }+ auto result = node_args_.insert(std::make_pair(name, onnxruntime::make_unique<NodeArg>(name, p_arg_type)));+ return *(result.first->second);+ }++#if !defined(ORT_MINIMAL_BUILD)+ /** Generate a unique name in this Graph for a NodeArg */+ std::string GenerateNodeArgName(const std::string& base_name);++ /** Generate a unique name in this Graph for a Node */+ std::string GenerateNodeName(const std::string& base_name);++ /** Copy a Node and add it to this Graph.+ @param other Node to copy+ @returns Reference to the Node that was created and added to this Graph.+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.+ */+ Node& AddNode(const Node& other);+#endif++#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)+ /** Add a Node to this Graph.+ @param name The Node name. Must be unique in this Graph.+ @param op_type The operator type. e.g. ONNX operator name.+ @param description Arbitrary description of the Node.+ @param input_args The explicit inputs to this Node.+ @param output_args The outputs from this Node.+ @param attributes Optional NodeAttributes to add.+ @param domain The domain for the op_type.+ @returns Reference to the new Node.+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.+ */+ Node& AddNode(const std::string& name,+ const std::string& op_type,+ const std::string& description,+ const std::vector<NodeArg*>& input_args,+ const std::vector<NodeArg*>& output_args,+ const NodeAttributes* attributes = nullptr,+ const std::string& domain = "");++ /** Remove a Node from this Graph and free it.+ The output edges of this specified node MUST have been removed before removing the node.+ The input edges of this specified node is removed while removing the node. The process of+ removing a node from a graph should be,+ 1. Remove out edges of this specified node.+ 2. Remove this specified node.+ 3. Add new input edges connected with all out nodes.+ @returns true if the node_index was valid+ @remarks Do not call AddNode and Remove Node concurrently as they are not thread-safe.+ */+ bool RemoveNode(NodeIndex node_index);++ /** Add an edge between two Nodes.+ @param src_node_index NodeIndex of source Node that is providing output to the destination Node.+ @param dst_node_index NodeIndex of destination Node that is receiving input from the source Node.+ @param src_arg_index node arg index of source node.+ @param dst_arg_index node arg index of destination node.+ */+ void AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_arg_index, int dst_arg_index);++ /** Remove an edge between two Nodes.+ @param src_node_index NodeIndex of source Node to remove an output edge from.+ @param dst_node_index NodeIndex of destination Node to remove an input edge from.+ @param src_arg_index node arg index of source node.+ @param dst_arg_index node arg index of destination node.+ */+ void RemoveEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_arg_index, int dst_arg_index);+#endif++#if !defined(ORT_MINIMAL_BUILD)+ /**+ Add a control edge between two Nodes in this Graph.+ The source Node does not produce output that is directly consumed by the destination Node, however the+ destination Node must execute after the source node. The control edge allows this ordering to occur.+ */+ bool AddControlEdge(NodeIndex src_node_index, NodeIndex dst_node_index);+#endif // !defined(ORT_MINIMAL_BUILD)++ /** Mark the Graph as needing Resolve() to be called.+ This should be done after modifying any aspect of the Graph that changes the Nodes or relationships between them. */+ Graph& SetGraphResolveNeeded() noexcept {+ graph_resolve_needed_ = true;+ return *this;+ }++ /** Gets flag indicating whether Graph::Resolve needs to be called before using the Graph. */+ bool GraphResolveNeeded() const noexcept {+ return graph_resolve_needed_;+ }++ /** Sets flag that Graph::graph_proto_ needs to be updated to reflect changes in the Graph. */+ Graph& SetGraphProtoSyncNeeded() noexcept {+ graph_proto_sync_needed_ = true;+ return *this;+ }++ /** Gets flag indicating whether Graph::graph_proto_ needs to be synchronized with this Graph instance. */+ bool GraphProtoSyncNeeded() const noexcept {+ return graph_proto_sync_needed_;+ }++ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,+ up to their source node/s.+ @param from NodeIndex values for a set of Nodes to traverse from.+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.+ @param leave Visit function invoked on the node after its parents have all been visited.+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.+ */+ void ReverseDFSFrom(const std::vector<NodeIndex>& from,+ const std::function<void(const Node*)>& enter,+ const std::function<void(const Node*)>& leave,+ const std::function<bool(const Node*, const Node*)>& comp = {}) const;++ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,+ up to their source node/s.+ @param from Set of Nodes to traverse from.+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.+ @param leave Visit function invoked on the node after its parents have all been visited.+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.+ */+ void ReverseDFSFrom(const std::vector<const Node*>& from,+ const std::function<void(const Node*)>& enter,+ const std::function<void(const Node*)>& leave,+ const std::function<bool(const Node*, const Node*)>& comp = {}) const;++ /** Performs a reverse depth-first search (DFS) traversal from a set of nodes, via their inputs,+ up to their source node/s.+ @param from Set of Nodes to traverse from.+ @param enter Visit function that will be invoked on a node when it is visited but its parents haven't been.+ @param leave Visit function invoked on the node after its parents have all been visited.+ @param stop Stop traversal from node n to input node p if stop(n, p) is true.+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.+ */+ void ReverseDFSFrom(const std::vector<const Node*>& from,+ const std::function<void(const Node*)>& enter,+ const std::function<void(const Node*)>& leave,+ const std::function<bool(const Node*, const Node*)>& comp,+ const std::function<bool(const Node*, const Node*)>& stop) const;++#if !defined(ORT_MINIMAL_BUILD)+ /** Performs topological sort with Kahn's algorithm on the graph/s.+ @param enter Visit function that will be invoked on a node when it is visited.+ @param comp Comparison function to stabilize the traversal order by making Node ordering deterministic.+ */+ void KahnsTopologicalSort(const std::function<void(const Node*)>& enter,+ const std::function<bool(const Node*, const Node*)>& comp) const;++#endif++ /** Gets the map of operator domains to their opset versions. */+ const std::unordered_map<std::string, int>& DomainToVersionMap() const noexcept {+ return domain_to_version_;+ }++#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)+ /**+ Create a single Node that will be the result of the a fusion of multiple nodes in this Graph.+ @param sub_graph A IndexSubGraph instance with details of the nodes to fuse.+ @param fused_node_name The name for the new Node.+ @returns Node with fused subgraph.+ @remarks As a new Graph instance for the fused nodes is not created, a GraphViewer can be constructed with the+ IndexedSubGraph information to provide a view of the subgraph. The original nodes are left in place+ while this is in use.+ Call FinalizeFuseSubGraph to remove them once the fused replacement node is fully created.+ */+ Node& BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);++ void FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node);+#endif++#if !defined(ORT_MINIMAL_BUILD)+ /** Gets the GraphProto representation of this Graph. */+ const ONNX_NAMESPACE::GraphProto& ToGraphProto();+ ONNX_NAMESPACE::GraphProto ToGraphProto() const;++ /** Gets the ISchemaRegistry instances being used with this Graph. */+ IOnnxRuntimeOpSchemaCollectionPtr GetSchemaRegistry() const;++ /**+ Create a single Function based Node that is the result of the a fusion of multiple nodes in this Graph.+ A new Graph instance will be created for the fused nodes.+ @param sub_graph A IndexSubGraph instance with details of the nodes to fuse. Ownership is transferred to the new Node+ @param fused_node_name The name for the new Node.+ @returns Function based Node with fused subgraph. The Node body will contain a Function instance.+ */+ Node& FuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);++ /**+ Directly insert the nodes in the function Node provided into this Graph.+ @param node Node with Node::Type of Node::Type::Fused+ @returns Status indicating success or providing an error message.+ */+ Status InlineFunction(Node& node);++ /** Initialize function body for the given node */+ void InitFunctionBodyForNode(Node& node);++ /** Mark a NodeArg name as coming from the outer scope when programmatically constructing a Graph that will+ be used as a GraphProto attribute in another Node..+ e.g. when creating a Graph instance that will be used as a subgraph in a control flow operator, it is necessary to+ define placeholder NodeArgs for outer scope values. This prevents these values from becoming explicit graph inputs+ when the Graph is resolved.+ */+ void AddOuterScopeNodeArg(const std::string& name) {+ ORT_IGNORE_RETURN_VALUE(outer_scope_node_arg_names_.insert(name));+ }++ /** Explicitly set graph inputs.+ @param inputs NodeArgs that represent complete graph inputs which need to be explicitly ordered.+ @remarks Note that the input order matters for subgraphs.+ */+ void SetInputs(const std::vector<const NodeArg*>& inputs);++ /** Explicitly set graph outputs.+ @param outputs NodeArgs that represent complete graph outputs which need to be explicitly ordered.+ @remarks Note that the output order matters for subgraphs.+ */+ void SetOutputs(const std::vector<const NodeArg*>& outputs);++ /** Sets the type of a NodeArg, replacing existing type/shape if any */+ void SetNodeArgType(NodeArg& arg, const onnx::TypeProto& type_proto);++ const Node* GetProducerNode(const std::string& node_arg_name) const {+ return GetProducerNodeImpl(*this, node_arg_name);+ }++ Node* GetMutableProducerNode(const std::string& node_arg_name) {+ return GetProducerNodeImpl(*this, node_arg_name);+ }++ void UpdateProducerNode(const std::string& node_arg_name, NodeIndex node_index) {+ auto iter = node_arg_to_producer_node_.find(node_arg_name);++ if (iter != node_arg_to_producer_node_.end()) {+ iter->second = node_index;+ } else {+ node_arg_to_producer_node_[node_arg_name] = node_index;+ }+ }++ std::vector<const Node*> GetConsumerNodes(const std::string& node_arg_name) const {+ return GetConsumerNodesImpl(*this, node_arg_name);+ }++ std::vector<Node*> GetMutableConsumerNodes(const std::string& node_arg_name) {+ return GetConsumerNodesImpl(*this, node_arg_name);+ }++ void UpdateConsumerNodes(const std::string& node_arg_name, const std::vector<Node*>& nodes) {+ auto iter = node_arg_to_consumer_nodes_.find(node_arg_name);+ if (iter != node_arg_to_consumer_nodes_.end()) {+ node_arg_to_consumer_nodes_.erase(node_arg_name);+ }+ for (Node* node : nodes) {+ node_arg_to_consumer_nodes_[node_arg_name].insert(node->Index());+ }+ }++ /** During constant folding it may become possible to infer the shape for a node.+ To avoid running a full Resolve allow an individual node to have the shape inferencing re-run.+ */+ Status UpdateShapeInference(Node& node);++ // Options to control Graph::Resolve.+ struct ResolveOptions {+ // Whether to override existing types with inferred types.+ bool override_types = false;+ // Names of initializers to keep even if unused (optional).+ const std::unordered_set<std::string>* initializer_names_to_preserve = nullptr;+ // Whether to set that no proto sync is required after resolving.+ // Useful for resolving right after loading from a GraphProto.+ bool no_proto_sync_required = false;+ };++ /**+ Resolve this Graph to ensure it is completely valid, fully initialized, and able to be executed.+ 1. Run through all validation rules.+ a. Node name and node output's names should be unique.+ b. Attribute match between node and op definition.+ c. Input/Output match between node and op definition.+ d. Graph is acyclic and sort nodes in topological order.+ 2. Check & Setup inner nodes' dependency.+ 3. Cleanup function definition lists.+ Note: the weights for training can't be cleaned during resolve.+ @returns common::Status with success or error information.+ */+ common::Status Resolve(const ResolveOptions& options);++ common::Status Resolve() {+ ResolveOptions default_options;+ return Resolve(default_options);+ }++ common::Status SaveToOrtFormat(flatbuffers::FlatBufferBuilder& builder,+ flatbuffers::Offset<onnxruntime::experimental::fbs::Graph>& fbs_graph) const;++#endif // !defined(ORT_MINIMAL_BUILD)++ /** Returns the Node containing the GraphProto for this Graph instance if IsSubgraph is true */+ const Node* ParentNode() const { return parent_node_; }++ /** Returns true if the name is for a value that is coming from outer scope */+ bool IsOuterScopeValue(const std::string& name) const {+ if (!parent_node_) return false;+ const auto& implicit_input_defs = parent_node_->ImplicitInputDefs();+ return std::any_of(implicit_input_defs.cbegin(), implicit_input_defs.cend(),+ [&name](const NodeArg* implicit_input) {+ return implicit_input->Name() == name;+ });+ }++#if !defined(ORT_MINIMAL_BUILD)+ /** Construct a Graph instance for a subgraph that is created from a GraphProto attribute in a Node.+ Inherits some properties from the parent graph.+ @param parent_graph The Graph containing the Node that has the GraphProto attribute.+ @param parent_node The Node that has the GraphProto attribute.+ @param subgraph_proto The GraphProto from the Node attribute.+ */+ Graph(Graph& parent_graph, const Node& parent_node, ONNX_NAMESPACE::GraphProto& subgraph_proto);+#endif++ virtual ~Graph();++#if defined(ENABLE_ORT_FORMAT_LOAD)+ static common::Status LoadFromOrtFormat(+ const onnxruntime::experimental::fbs::Graph& fbs_graph, const Model& owning_model,+ const std::unordered_map<std::string, int>& domain_to_version,+ const logging::Logger& logger, std::unique_ptr<Graph>& graph);++ // deserialize a subgraph+ static Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Graph& fbs_graph,+ Graph& parent_graph, const Node& parent_node,+ const logging::Logger& logger, std::unique_ptr<Graph>& graph);+#endif+ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Graph);++ // This friendship relationship should only be used to call Graph::Graph and+ // Graph::LoadGraph All other access should be via the public API.+ friend class Model;++ Graph() = delete;++ // Create empty Graph instance to re-create from ORT format serialized data.+ Graph(const Model& owning_model,+ const std::unordered_map<std::string, int>& domain_to_version,+ Graph* parent_graph, const Node* parent_node,+ const logging::Logger& logger);++#if defined(ENABLE_ORT_FORMAT_LOAD)+ // Populate Graph instance from ORT format serialized data.+ common::Status LoadFromOrtFormat(const onnxruntime::experimental::fbs::Graph& fbs_graph);+#endif++#if !defined(ORT_MINIMAL_BUILD)+ // Constructor: Given a <GraphProto> loaded from model file, construct+ // a <Graph> object. Used by Model to create a Graph instance.+ Graph(const Model& owning_model,+ ONNX_NAMESPACE::GraphProto* graph_proto,+ const std::unordered_map<std::string, int>& domain_to_version,+ Version ir_version,+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry,+ const logging::Logger& logger);++ // internal use by the Graph class only+ Graph(const Model& owning_model,+ ONNX_NAMESPACE::GraphProto* graph_proto,+ const std::unordered_map<std::string, int>& domain_to_version,+ Version ir_version,+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry,+ Graph* parent_graph,+ const Node* parent_node,+ const logging::Logger& logger);++ void InitializeStateFromModelFileGraphProto();++ // Add node with specified <node_proto>.+ Node& AddNode(const ONNX_NAMESPACE::NodeProto& node_proto,+ const ArgNameToTypeMap& name_to_type);++#endif++ Version IrVersion() const noexcept {+ return ir_version_;+ }++ Graph& GraphResolveNeeded(bool needed) noexcept {+ graph_resolve_needed_ = needed;+ return *this;+ }++ Graph& GraphProtoSyncNeeded(bool needed) noexcept {+ graph_proto_sync_needed_ = needed;+ return *this;+ }++ // During the Resolve of a Graph it is necessary to recursively descend into subgraphs (created from GraphProto+ // Node attributes in the Graph) if present.+ // The ResolveContext holds the collection of values for the current Graph instance, be it the main graph+ // or a subgraph, so that the various operations that are part of the Resolve can work iteratively or+ // recursively as needed.+ struct ResolveContext {+ ResolveContext() = default;++ std::unordered_map<std::string, std::pair<Node*, int>> output_args;+ std::unordered_set<std::string> inputs_and_initializers;+ std::unordered_set<std::string> outer_scope_node_args;+ std::unordered_map<std::string, NodeIndex> node_name_to_index;+ std::unordered_set<Node*> nodes_with_subgraphs;++ void Clear() {+ output_args.clear();+ inputs_and_initializers.clear();+ outer_scope_node_args.clear();+ node_name_to_index.clear();+ nodes_with_subgraphs.clear();+ }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ResolveContext);+ };++ // Initialize all the graph inputs, initializers and outputs+ common::Status InitInputsInitializersOutputs();++ // Initialize overridable initializers container+ void ComputeOverridableInitializers();++#if !defined(ORT_MINIMAL_BUILD)+ // Build and verify node connection (edges).+ // Verify NodeArg name/type/shape matching correctly.+ common::Status BuildConnections(std::unordered_set<std::string>& outer_scope_node_args_consumed);++ common::Status VerifyNoDuplicateName();++ // Check whether <*this> graph is acyclic while performing a topological sort.+ // Depth-first going from bottom up through the graph and checking whether there are any back edges.+ // NodesInTopologicalOrder is updated with the nodes' indexes in topological+ // order if <Status> returned is "OK", otherwise it's undefined.+ common::Status PerformTopologicalSortAndCheckIsAcyclic();++ common::Status PerformTypeAndShapeInferencing(const ResolveOptions& options);++ // Recursively find all subgraphs including nested subgraphs+ void FindAllSubgraphs(std::vector<Graph*>& subgraphs);++ // Iterate this Graph instance and all subgraphs, calling the provided function for each.+ common::Status ForThisAndAllSubgraphs(const std::vector<Graph*>& subgraphs, std::function<Status(Graph&)> func);++ common::Status InferAndVerifyTypeMatch(Node& node, const ONNX_NAMESPACE::OpSchema& op, const ResolveOptions& options);++ // perform type and shape inferencing on the subgraph and Resolve to validate+ static common::Status InferAndVerifySubgraphTypes(const Node& node, Graph& subgraph,+ const std::vector<const ONNX_NAMESPACE::TypeProto*>& input_types,+ std::vector<const ONNX_NAMESPACE::TypeProto*>& output_types,+ const Graph::ResolveOptions& options);++ // Apply type-inference and type-checking to all inputs and initializers:+ common::Status TypeCheckInputsAndInitializers();++ // Compute set of input and initializer names and checking for duplicate names+ common::Status VerifyInputAndInitializerNames();++ // Infer and set type information across <*this> graph if needed, and verify type/attribute+ // information matches between node and op.+ common::Status VerifyNodeAndOpMatch(const ResolveOptions& options);++ // Set graph inputs/outputs when resolving a graph..+ common::Status SetGraphInputsOutputs();++ // recursively accumulate and set the outer scope node args in the resolve context for all subgraphs+ // so they can be used to resolve outer scope dependencies when running BuildConnections for the subgraphs.+ common::Status SetOuterScopeNodeArgs(const std::unordered_set<std::string>& outer_scope_node_args);++ // Clear all unused initializers+ void CleanUnusedInitializers(const std::unordered_set<std::string>* initializer_names_to_preserve = nullptr);++ std::vector<NodeArg*> CreateNodeArgs(const google::protobuf::RepeatedPtrField<std::string>& names,+ const ArgNameToTypeMap& name_to_type_map);++ void ToGraphProtoInternal(ONNX_NAMESPACE::GraphProto& graph_proto) const;++ template <typename TInstance>+ static auto GetProducerNodeImpl(+ TInstance& instance, const std::string& node_arg_name) -> decltype(instance.GetNode(0)) {+ auto iter = instance.node_arg_to_producer_node_.find(node_arg_name);+ if (iter != instance.node_arg_to_producer_node_.end()) {+ auto node_index = iter->second;+ return instance.GetNode(node_index);+ }+ return nullptr;+ }++ template <typename TInstance>+ static auto GetConsumerNodesImpl(+ TInstance& instance, const std::string& node_arg_name) -> std::vector<decltype(instance.GetNode(0))> {+ std::vector<decltype(instance.GetNode(0))> results;+ auto iter = instance.node_arg_to_consumer_nodes_.find(node_arg_name);+ if (iter != instance.node_arg_to_consumer_nodes_.end()) {+ results.reserve(iter->second.size());+ for (auto node_index : iter->second) {+ results.push_back(instance.GetNode(node_index));+ }+ }+ return results;+ }++#endif // !defined(ORT_MINIMAL_BUILD)++#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)+ gsl::not_null<Node*> AllocateNode();++ // Release the node.+ // @returns false if node_index was invalid.+ bool ReleaseNode(NodeIndex node_index);++ Node& CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);+#endif++ Node* NodeAtIndexImpl(NodeIndex node_index) const {+ // if we are trying to access a node that doesn't exist there's (most+ // likely) either a logic issue or a graph consistency/correctness issue.+ // use ORT_ENFORCE to prove that or uncover scenarios where we actually+ // expect attempts to retrieve a non-existent node.+ ORT_ENFORCE(node_index < nodes_.size(), "Validating no unexpected access using an invalid node_index. Got:",+ node_index, " Max:", nodes_.size());+ return nodes_[node_index].get();+ }++ const Model& owning_model_;++ // GraphProto to store name, version, initializer.+ // When serializing <*this> Graph to a GraphProto, the nodes and+ // functions in <Graph> will also be fed into <graph_proto_> so that+ // it's consistent with <*this> graph.+ // This pointer is owned by parent model.+ ONNX_NAMESPACE::GraphProto* graph_proto_;++ // GraphProto that provides storage for the ONNX proto types deserialized from a flexbuffer/flatbuffer+ ONNX_NAMESPACE::GraphProto deserialized_proto_data_;++ InitializedTensorSet name_to_initial_tensor_;++ std::unordered_set<std::reference_wrapper<const std::string>,+ std::hash<std::string>, std::equal_to<std::string>>+ sparse_tensor_names_;++#if !defined(ORT_MINIMAL_BUILD)+ IOnnxRuntimeOpSchemaCollectionPtr schema_registry_;++ std::vector<std::unique_ptr<onnxruntime::Function>> function_container_;+#endif++ // Graph nodes.+ // Element in <nodes_> may be nullptr due to graph optimization.+ std::vector<std::unique_ptr<Node>> nodes_;++ // Wrapper of Graph nodes to provide iteration services that hide nullptr entries+ GraphNodes iterable_nodes_{nodes_};++ // Number of nodes.+ // Normally this is smaller than the size of <m_nodes>, as some+ // elements in <m_nodes> may be removed when doing graph optimization,+ // or some elements may be merged, etc.+ int num_of_nodes_ = 0;++ // A flag indicates whether <*this> graph needs to be resolved.+ bool graph_resolve_needed_ = false;++ bool graph_proto_sync_needed_ = false;++ // The topological order of node index used to do node and op match verification temporarily.+ std::vector<NodeIndex> nodes_in_topological_order_;++ // Full list of graph inputs. Matches number and order of inputs in the GraphProto.+ std::vector<const NodeArg*> graph_inputs_including_initializers_;+ bool graph_inputs_manually_set_ = false;++ // Graph inputs excluding initializers.+ std::vector<const NodeArg*> graph_inputs_excluding_initializers_;++ // Overridable Initializers. The difference between graph_inputs_including_initializers_+ // and graph_inputs_excluding_initializers_+ std::vector<const NodeArg*> graph_overridable_initializers_;++ // Graph outputs.+ std::vector<const NodeArg*> graph_outputs_;+ bool graph_outputs_manually_set_ = false;++ // Graph value_info.+ std::vector<const NodeArg*> value_info_;++ // All node args owned by <*this> graph. Key is node arg name.+ std::unordered_map<std::string, std::unique_ptr<NodeArg>> node_args_;++#if !defined(ORT_MINIMAL_BUILD)+ int name_generator_ = 0;++ // Strings which have been used as node names.+ // New node name should not conflict with this set.+ std::unordered_set<std::string> generated_node_names_;++ // Strings which have been used as node_arg names.+ // New node_arg name should not conflict this this set.+ std::unordered_set<std::string> generated_node_arg_names_;++ // node arg to its producer node+ std::unordered_map<std::string, NodeIndex> node_arg_to_producer_node_;++ // node arg to its consumer nodes+ std::unordered_map<std::string, std::unordered_set<NodeIndex>> node_arg_to_consumer_nodes_;++#endif // !defined(ORT_MINIMAL_BUILD)++ const std::unordered_map<std::string, int> domain_to_version_;++ // Model IR version.+ Version ir_version_{ONNX_NAMESPACE::Version::IR_VERSION};++ // Is model using latest ONNX opset+ bool using_latest_onnx_opset_{false};++ ResolveContext resolve_context_;++ // the parent graph if this is a subgraph.+ Graph* parent_graph_;+ // the node containing the graph if parent_graph_ is not nullptr+ const Node* parent_node_;++ // NodeArgs that come from outer scope. Used when building a graph so that+ // these don't get recorded as graph inputs in the GraphProto.+ std::unordered_set<std::string> outer_scope_node_arg_names_;++ // number of times Resolve has run.+ int num_resolves_ = 0;++ const logging::Logger& logger_;++ // distinguishes between graph loaded from model file and graph created from scratch+ const bool is_loaded_from_model_file_;+};++#if !defined(ORT_MINIMAL_BUILD)+std::ostream& operator<<(std::ostream& out, const Graph& graph);+#endif++} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/graph_nodes.h b/include/onnxruntime/core/graph/graph_nodes.hnew file mode 100644index 0000000..422fe95--- /dev/null+++ b/include/onnxruntime/core/graph/graph_nodes.h@@ -0,0 +1,182 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <memory>+#include <type_traits>+#include <vector>++namespace onnxruntime {++class Node;++/**+Class to filter out null entries from either a vector of unique_ptr<Node> or a vector of [const] Node* and+provide an iterator interface that returns [const] Node& for the valid entries.+*/+template <typename TNodesContainer>+class ValidNodes {+ public:+ template <typename TIterator>+ class NodeIterator;++ // optional filtering function to return a subset of nodes+ using NodeFilterFunc = std::function<bool(NodeIndex)>;++ /**+ Construct a ValidNodes instance to provide iteration over all valid nodes in the TNodesCollection+ @param[in] nodes Nodes to iterate, skipping invalid entries.+ */+ explicit ValidNodes(TNodesContainer& nodes) noexcept : nodes_(&nodes) {}++ explicit ValidNodes(TNodesContainer& nodes, NodeFilterFunc&& filter_node_fn) noexcept+ : nodes_(&nodes), filter_node_fn_{std::move(filter_node_fn)} {}++ using ConstNodeIterator = NodeIterator<typename TNodesContainer::const_iterator>;+ using MutableNodeIterator = NodeIterator<typename TNodesContainer::iterator>;+ using ConstReverseNodeIterator = NodeIterator<typename TNodesContainer::const_reverse_iterator>;++ ConstNodeIterator cbegin() const noexcept {+ return {nodes_->cbegin(), nodes_->cend(), filter_node_fn_};+ }++ ConstNodeIterator cend() const noexcept {+ return {nodes_->cend(), nodes_->cend(), filter_node_fn_};+ }++ ConstNodeIterator begin() const noexcept {+ return cbegin();+ }++ ConstNodeIterator end() const noexcept {+ return cend();+ }++ ConstReverseNodeIterator rbegin() const noexcept {+ return {nodes_->crbegin(), nodes_->crend(), filter_node_fn_};+ }++ ConstReverseNodeIterator rend() const noexcept {+ return {nodes_->crend(), nodes_->crend(), filter_node_fn_};+ }++ // we only allow mutable access if the container is non-const.+ // we need to templatize the functions for enable_if to work at this level, but mandate T2 being TNodesContainer+ template <typename T2 = TNodesContainer>+ typename std::enable_if<!std::is_const<T2>::value, MutableNodeIterator>::type begin() noexcept {+ static_assert(std::is_same<T2, TNodesContainer>::value, "Explicit specialization is not allowed");+ return MutableNodeIterator(nodes_->begin(), nodes_->end(), filter_node_fn_);+ }++ template <typename T2 = TNodesContainer>+ typename std::enable_if<!std::is_const<T2>::value, MutableNodeIterator>::type end() noexcept {+ static_assert(std::is_same<T2, TNodesContainer>::value, "Explicit specialization is not allowed");+ return MutableNodeIterator(nodes_->end(), nodes_->end(), filter_node_fn_);+ }++ bool empty() const noexcept { return nodes_->empty(); }++ /**+ @class NodeIterator+ Iterator to provide const and non-const access to valid Node instances in a Graph.+ @remarks Skips invalid nodes.+ */+ template <typename TIterator>+ class NodeIterator {+ // get the type being returned by the iterator. can't use TIterator::value_type as that is always non-const+ using IterType = typename std::remove_reference<typename std::iterator_traits<TIterator>::reference>::type;+ // and determine what we will return based on its constness+ using T = typename std::conditional<std::is_const<IterType>::value,+ const Node, // return const Node if this is a const iterator+ Node>::type; // else return Node++ public:+ using iterator_category = std::input_iterator_tag;+ using value_type = T;+ using difference_type = typename TIterator::difference_type;+ using pointer = T*;+ using reference = T&;+ using const_reference = const T&;++ /** Construct a NodeInterator and move to the first valid node. */+ NodeIterator<TIterator>(const TIterator current, const TIterator end, const NodeFilterFunc& filter_fn) noexcept+ : current_{current}, end_{end}, apply_filter_{filter_fn != nullptr}, filter_func_{&filter_fn} {+ // skip to next valid node, stopping at end if none are found+ while (current_ < end && (*current_ == nullptr ||+ (apply_filter_ && (*filter_func_)((*current_)->Index()) == true))) {+ ++current_;+ }+ }++ bool operator==(const NodeIterator<TIterator>& other) const noexcept {+ return (current_ == other.current_);+ }++ bool operator!=(const NodeIterator<TIterator>& other) const noexcept {+ return (current_ != other.current_);+ }++ void operator++() {+ if (current_ < end_) {+ while (++current_ != end_) {+ if (*current_ != nullptr && (!apply_filter_ || (*filter_func_)((*current_)->Index()) == false))+ break;+ }+ }+ }++ NodeIterator<TIterator> operator++(int) {+ NodeIterator<TIterator> tmp{*this};+ ++(*this);++ return tmp;+ }++ /** Return the current Node&. This will be const if the iterator was returned from a const GraphNodes instance. */+ reference operator*() {+ // if iterator is valid we always have a non-nullptr node+ // if this is a nullptr we're at end_ and this shouldn't be being called+ return **current_;+ }++ pointer operator->() {+ return current_->get();+ }++ private:+ TIterator current_;+ TIterator end_;+ bool apply_filter_; // store whether filter_func_ is not nullptr and contains a callable+ const NodeFilterFunc* filter_func_; // store as pointer so iterator is copyable+ };++ private:+ gsl::not_null<TNodesContainer*> nodes_; // always set by ctor++ // no filtering if not set. this instance owns the filter func if set.+ NodeFilterFunc filter_node_fn_;+};++/**+Class that provides iteration over all valid nodes in the Graph.+*/+class GraphNodes : public ValidNodes<std::vector<std::unique_ptr<Node>>> {+ public:+ GraphNodes(std::vector<std::unique_ptr<Node>>& nodes) : ValidNodes(nodes) {+ }+};++// Variant that only ever allows const access to nodes and optionally allows filtering of the nodes.+class ConstGraphNodes : public ValidNodes<const std::vector<std::unique_ptr<Node>>> {+ public:+ ConstGraphNodes(const std::vector<std::unique_ptr<Node>>& nodes) : ValidNodes(nodes) {+ }++ ConstGraphNodes(const std::vector<std::unique_ptr<Node>>& nodes,+ GraphNodes::NodeFilterFunc&& filter_func)+ : ValidNodes(nodes, std::move(filter_func)) {+ }+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/graph_viewer.h b/include/onnxruntime/core/graph/graph_viewer.hnew file mode 100644index 0000000..b2d9ee2--- /dev/null+++ b/include/onnxruntime/core/graph/graph_viewer.h@@ -0,0 +1,196 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/graph/graph.h"+#include "core/framework/session_options.h"+#include <unordered_set>++namespace onnxruntime {+class Function;+struct IndexedSubGraph;+} // namespace onnxruntime++namespace onnxruntime {++// use value-based compare to make sure transformer output order is consistent+struct NodeCompare {+ bool operator()(const Node* n1, const Node* n2) const;+};++/**+@class GraphViewer+Class that provides a read-only view of the Graph.+@remarks If the underlying Graph is changed, GetNodesInTopologicalOrder and GetRootNodes may become invalid.+*/+class GraphViewer {+ public:+ /**+ Construct a GraphViewer from the provided Graph instance.+ */+ explicit GraphViewer(const Graph& graph);++ /**+ Construct a GraphViewer from the provided Graph instance, filtering to the nodes specified in the IndexedSubGraph+ */+ explicit GraphViewer(const Graph& graph, const IndexedSubGraph& filter_info);++ /** Gets the Graph name. */+ const std::string& Name() const noexcept;++ /** Gets the Graph description. */+ const std::string& Description() const noexcept;++ /** Gets the path of the owning model if any **/+ const Path& ModelPath() const noexcept { return graph_->ModelPath(); }++ /**+ Gets a tensor created from an initializer.+ @param tensor_name The tensor name+ @param[out] value Sets the pointer to the TensorProto if found, or nullptr if not.+ @returns True if found. False if not.+ */+ bool GetInitializedTensor(const std::string& tensor_name, const ONNX_NAMESPACE::TensorProto*& value) const;++ /** Returns true if an initializer value can be overridden by a graph input with the same name. */+ bool CanOverrideInitializer() const noexcept;++ /**+ Gets the Graph inputs, excluding initializers.+ @returns Collection of NodeArg pointers for the graph inputs, excluding inputs that have matching initializers.+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.+ Inputs are for filter_info_ if set.+ */+ const std::vector<const NodeArg*>& GetInputs() const noexcept;++ /**+ Gets the Graph inputs, including any initializers.+ @returns Collection of NodeArg pointers for all the graph inputs.+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.+ Inputs are for filter_info_ if set.+ */+ const std::vector<const NodeArg*>& GetInputsIncludingInitializers() const noexcept;++ /**+ Gets the Graph outputs.+ @returns Collection of NodeArg pointers for all the graph outputs.+ @remarks No nullptr values in the returned collection. The order will be the same as in the GraphProto.+ Outputs are for filter_info_ if set.+ */+ const std::vector<const NodeArg*>& GetOutputs() const noexcept;++ /** Gets all ValueInfo NodeArg instances in the Graph.+ @remarks NOT filtered using filter_info_.+ */+ const std::vector<const NodeArg*>& GetValueInfo() const noexcept;++ /**+ Gets the Node instance at the specified index.+ @param node_index Index to retrieve Node from.+ @remarks May return nullptr if index no longer points to a valid node due to the node being freed, or if+ node is excluded by filter_info_.+ */+ const Node* GetNode(NodeIndex node_index) const;++ /** Gets an iterator over all the valid Nodes in the Graph.+ @remarks Nodes are filtered using filter_info_ if set.+ */+ const ConstGraphNodes& Nodes() const noexcept;++ /** Gets the number of valid nodes in the Graph.+ @remarks Returns the number of nodes in filter_info_ if set.+ */+ int NumberOfNodes() const noexcept;++ /** Gets the maximum NodeIndex value used by Nodes in the Graph. */+ int MaxNodeIndex() const noexcept;++ /** Gets the NodeIndex values for the Graph nodes, sorted into topological order.+ @remarks Filtered using filter_info_ if set.+ */+ const std::vector<NodeIndex>& GetNodesInTopologicalOrder(ExecutionOrder order = ExecutionOrder::DEFAULT) const;++ /**+ Gets the NodeIndex values for the root nodes in the Graph.+ The root nodes are the topmost nodes in the Graph that receive inputs from the Graph inputs+ and no other nodes in the Graph.+ @remarks Not supported if filter_info_ is set.+ */+ const std::vector<NodeIndex>& GetRootNodes() const;++ /** Gets all tensors created from initializers. */+ const InitializedTensorSet& GetAllInitializedTensors() const noexcept;++ /**+ Gets the NodeArg instance for the given name.+ @returns A NodeArg if found, a nullptr if not.+ */+ const NodeArg* GetNodeArg(const std::string& name) const;++ /** Gets the map of operator domains to their opset versions. */+ const std::unordered_map<std::string, int>& DomainToVersionMap() const noexcept {+ return graph_->DomainToVersionMap();+ }++ /** Checks if this is a Subgraph */+ bool IsSubgraph() const;++ /** Get the internal graph*/+ const Graph& GetGraph() const { return *graph_; }++ /**+ returns true if 'name' is an initializer, and is constant and cannot be overridden at runtime.+ @param check_outer_scope If true and the 'graph_' is a subgraph, check parent graph/s for 'name'+ if the name is not found in 'graph_'.+ */+ bool IsConstantInitializer(const std::string& name, bool check_outer_scope) const;++ /** Get the Node containing this Graph if IsSubgraph is true. Returns nullptr otherwise. */+ const Node* ParentNode() const noexcept { return graph_->ParentNode(); }++#if !defined(ORT_MINIMAL_BUILD)+ /** Get the consumer nodes of a node arg */+ std::vector<const Node*> GetConsumerNodes(const std::string& node_arg_name) const {+ return graph_->GetConsumerNodes(node_arg_name);+ }++ /** Get the producer node of a node arg */+ const Node* GetProducerNode(const std::string& node_arg_name) const {+ return graph_->GetProducerNode(node_arg_name);+ }+#endif++ /** Get the filter info that restricts the graph viewer to a subset of nodes if set.+ @returns Filter info or nullptr+ */+ const IndexedSubGraph* GetFilterInfo() const { return filter_info_; }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer);+ GraphViewer(const Graph& graph, const IndexedSubGraph* filter_info);++ const Graph* graph_;+ ConstGraphNodes graph_nodes_;++ // The NodeIndex values of the graph nodes sorted in topological order.+ std::vector<NodeIndex> nodes_in_topological_order_;++#if !defined(ORT_MINIMAL_BUILD)+ // The NodeIndex values of the graph nodes sorted in topological order with priority.+ std::vector<NodeIndex> nodes_in_topological_order_with_priority_;+#endif++ // Graph root nodes.+ std::vector<NodeIndex> root_nodes_;++ // if we're limiting the view to an IndexedSubGraph we need to create a few pieces of infrastructure that would+ // usually come from the full graph+ const IndexedSubGraph* filter_info_{nullptr};+ std::unordered_set<NodeIndex> filtered_node_indices_;+ std::vector<const NodeArg*> filtered_node_inputs_;+ std::vector<const NodeArg*> filtered_node_inputs_including_initializers_;+ std::vector<const NodeArg*> filtered_node_outputs_;+ InitializedTensorSet filtered_initializers_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/indexed_sub_graph.h b/include/onnxruntime/core/graph/indexed_sub_graph.hnew file mode 100644index 0000000..b0485c4--- /dev/null+++ b/include/onnxruntime/core/graph/indexed_sub_graph.h@@ -0,0 +1,61 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <memory>+#include <string>+#include <vector>++#include "core/graph/basic_types.h"+#include "core/graph/onnx_protobuf.h"++namespace onnxruntime {++class OpKernel;+class OpKernelInfo;++/**+@class IndexedSubGraph++Class containing information about a subgraph of Nodes from a Graph.+It contains a NodeIndex array of the Nodes covered by the subgraph,+and the meta definition needed for representing this subgraph as a FunctionProto,+which could be serialized/saved to a model file.+*/+struct IndexedSubGraph {+ struct MetaDef {+ std::string name; ///< Name of customized SubGraph/FunctionProto+ std::string domain; ///< Domain of customized SubGraph/FunctionProto+ int since_version; ///< Since version of customized SubGraph/FunctionProto.++ ONNX_NAMESPACE::OperatorStatus status; ///< Status of customized SubGraph/FunctionProto.++ std::vector<std::string> inputs; ///< Inputs of customized SubGraph/FunctionProto.+ std::vector<std::string> outputs; ///< Outputs of customized SubGraph/FunctionProto.+ NodeAttributes attributes; ///< Attributes of customized SubGraph/FunctionProto.++ std::string doc_string; ///< Doc string of customized SubGraph/FunctionProto.+ };++ /** Nodes covered by this subgraph. The NodeIndex values are from the parent Graph.*/+ std::vector<onnxruntime::NodeIndex> nodes;++ /** Set the meta definition needed to represent this subgraph as a FunctionProto+ It's needed IF AND ONLY IF there are multiple indexes contained in #nodes. */+ void SetMetaDef(std::unique_ptr<MetaDef>&& meta_def_) {+ meta_def = std::move(meta_def_);+ }++ /** Gets the meta definition needed to represent this subgraph as a FunctionProto.+ @returns MetaDef instance if it has been set. nullptr if not. */+ const MetaDef* GetMetaDef() const {+ return meta_def.get();+ }++ private:+ // subgraph meta definition.+ std::unique_ptr<MetaDef> meta_def;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/node_arg.h b/include/onnxruntime/core/graph/node_arg.hnew file mode 100644index 0000000..eeb53d2--- /dev/null+++ b/include/onnxruntime/core/graph/node_arg.h@@ -0,0 +1,116 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/graph/basic_types.h"+#include "core/graph/onnx_protobuf.h"++namespace onnxruntime {++// Node argument definition, for both input and output,+// including arg name, arg type (contains both type and shape).+//+// Design Question: in my opinion, shape should not be part of type.+// We may align the protobuf design with our operator registry interface,+// which has type specified for each operator, but no shape. Well, shape+// should be inferred with a separate shape inference function given+// input shapes, or input tensor data sometimes.+// With shape as part of type (current protobuf design),+// 1) we'll have to split the "TypeProto" into type and shape in this internal+// representation interface so that it could be easily used when doing type+// inference and matching with operator registry.+// 2) SetType should be always called before SetShape, otherwise, SetShape()+// will fail. Because shape is located in a TypeProto.+// Thoughts?+//++/**+@class NodeArg+Class representing a data type that is input or output for a Node, including the shape if it is a Tensor.+*/+class NodeArg {+ public:+ /**+ Construct a new NodeArg.+ @param name The name to use.+ @param p_arg_type Optional TypeProto specifying type and shape information.+ */+ NodeArg(const std::string& name,+ const ONNX_NAMESPACE::TypeProto* p_arg_type);++ NodeArg(NodeArg&&) = default;+ NodeArg& operator=(NodeArg&& other) = default;++ /** Gets the name. */+ const std::string& Name() const noexcept;++ /** Gets the data type. */+ ONNX_NAMESPACE::DataType Type() const noexcept;++ /** Gets the TypeProto+ @returns TypeProto if type is set. nullptr otherwise. */+ const ONNX_NAMESPACE::TypeProto* TypeAsProto() const noexcept;++ /** Gets the shape if NodeArg is for a Tensor.+ @returns TensorShapeProto if shape is set. nullptr if there's no shape specified. */+ const ONNX_NAMESPACE::TensorShapeProto* Shape() const;++ /** Return an indicator.+ @returns true if NodeArg is a normal tensor with a non-empty shape or a scalar with an empty shape. Otherwise, returns false. */+ bool HasTensorOrScalarShape() const;++#if !defined(ORT_MINIMAL_BUILD)+ /** Sets the shape.+ @remarks Shape can only be set if the TypeProto was provided to the ctor, or #SetType has been called,+ as the shape information is stored as part of TypeProto. */+ void SetShape(const ONNX_NAMESPACE::TensorShapeProto& shape);++ /** Clears shape info.+ @remarks If there is a mismatch during shape inferencing that can't be resolved the shape info may be removed. */+ void ClearShape();++ /** Validate and merge type [and shape] info from input_type.+ @param strict If true, the shape update will fail if there are incompatible values.+ If false, will be lenient and merge only shape info that can be validly processed.+ @param override_types If true, resolve the two inputs or two outputs type when different+ @returns Success unless there is existing type or shape info that can't be successfully updated. */+ common::Status UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& input_type, bool strict, bool override_types, const logging::Logger& logger);++ /** Validate and merge type [and shape] info from node_arg.+ @param strict If true, the shape update will fail if there are incompatible values.+ If false, will be lenient and merge only shape info that can be validly processed.+ @param override_types If true, resolve the two inputs or two outputs type when different+ @returns Success unless there is existing type or shape info that can't be successfully updated. */+ common::Status UpdateTypeAndShape(const NodeArg& node_arg, bool strict, bool override_types, const logging::Logger& logger);++#endif // !defined(ORT_MINIMAL_BUILD)++ /** Gets this NodeArg as a ValueInfoProto. */+ const NodeArgInfo& ToProto() const noexcept { return node_arg_info_; }++ /** Gets a flag indicating whether this NodeArg exists or not.+ Optional inputs are allowed in ONNX and an empty #Name represents a non-existent input argument. */+ bool Exists() const noexcept;++ private:+ ORT_DISALLOW_COPY_AND_ASSIGNMENT(NodeArg);+ friend class Graph;++ NodeArg(NodeArgInfo&& node_arg_info);++#if !defined(ORT_MINIMAL_BUILD)+ void SetType(ONNX_NAMESPACE::DataType p_type);+ void SetType(const ONNX_NAMESPACE::TypeProto& type_proto);+#endif++ // Node arg PType.+ ONNX_NAMESPACE::DataType type_;++ // Node arg name, type and shape.+ NodeArgInfo node_arg_info_;++ // Flag indicates whether <*this> node arg exists or not.+ bool exists_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/graph/onnx_protobuf.h b/include/onnxruntime/core/graph/onnx_protobuf.hnew file mode 100644index 0000000..23a601e--- /dev/null+++ b/include/onnxruntime/core/graph/onnx_protobuf.h@@ -0,0 +1,50 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+//TODO(): delete this file from public interface+#ifdef __GNUC__+#include "onnxruntime_config.h"+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wignored-qualifiers"+#pragma GCC diagnostic ignored "-Wunused-parameter"+#ifdef HAS_DEPRECATED_DECLARATIONS+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"+#endif+#else+#pragma warning(push)+#pragma warning(disable : 4018) /*'expression' : signed/unsigned mismatch */+#pragma warning(disable : 4065) /*switch statement contains 'default' but no 'case' labels*/+#pragma warning(disable : 4100)+#pragma warning(disable : 4146) /*unary minus operator applied to unsigned type, result still unsigned*/+#pragma warning(disable : 4127)+#pragma warning(disable : 4244) /*'conversion' conversion from 'type1' to 'type2', possible loss of data*/+#pragma warning(disable : 4251) /*'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2'*/+#pragma warning(disable : 4267) /*'var' : conversion from 'size_t' to 'type', possible loss of data*/+#pragma warning(disable : 4305) /*'identifier' : truncation from 'type1' to 'type2'*/+#pragma warning(disable : 4307) /*'operator' : integral constant overflow*/+#pragma warning(disable : 4309) /*'conversion' : truncation of constant value*/+#pragma warning(disable : 4334) /*'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?)*/+#pragma warning(disable : 4355) /*'this' : used in base member initializer list*/+#pragma warning(disable : 4506) /*no definition for inline function 'function'*/+#pragma warning(disable : 4800) /*'type' : forcing value to bool 'true' or 'false' (performance warning)*/+#pragma warning(disable : 4996) /*The compiler encountered a deprecated declaration.*/+#pragma warning(disable : 6011) /*Dereferencing NULL pointer*/+#pragma warning(disable : 6387) /*'value' could be '0'*/+#pragma warning(disable : 26495) /*Variable is uninitialized.*/+#endif++#if !defined(ORT_MINIMAL_BUILD)+#include "onnx/defs/schema.h"+#else+#include "onnx/defs/data_type_utils.h"+#endif++#include "onnx/onnx_pb.h"+#include "onnx/onnx-operators_pb.h"++#ifdef __GNUC__+#pragma GCC diagnostic pop+#else+#pragma warning(pop)+#endifdiff --git a/include/onnxruntime/core/graph/schema_registry.h b/include/onnxruntime/core/graph/schema_registry.hnew file mode 100644index 0000000..0c7b0ce--- /dev/null+++ b/include/onnxruntime/core/graph/schema_registry.h@@ -0,0 +1,158 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#include "core/graph/constants.h"+#include "core/common/common.h"+#include "core/common/status.h"+#include "core/graph/onnx_protobuf.h"+#include "core/platform/ort_mutex.h"+#include <mutex>+#include <deque>+#include "sstream"++namespace onnxruntime {+using OpName_Domain_Version_Schema_Map = std::unordered_map<+ std::string,+ std::unordered_map<std::string, std::map<ONNX_NAMESPACE::OperatorSetVersion, ONNX_NAMESPACE::OpSchema>>>;++/**+@struct SchemaRegistryVersion+onnxruntime schema registry is a supplement to the built-in ONNX schema.+Every schema registry represent a collection of schema deltas from baseline_opset_version to opset_version+*/+struct SchemaRegistryVersion {+ int baseline_opset_version;+ int opset_version;+};++using DomainToVersionMap = std::unordered_map<std::string, int>;+using DomainToVersionRangeMap = std::unordered_map<std::string, SchemaRegistryVersion>;++class IOnnxRuntimeOpSchemaCollection : public ONNX_NAMESPACE::ISchemaRegistry {+ public:+ virtual DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const = 0;++ using ISchemaRegistry::GetSchema;++ const ONNX_NAMESPACE::OpSchema* GetSchema(const std::string& key, const int maxInclusiveVersion,+ const std::string& domain) const final {+ const ONNX_NAMESPACE::OpSchema* latest_schema = nullptr;+ int earliest_opset_where_unchanged = std::numeric_limits<int>::max();+ GetSchemaAndHistory(key, maxInclusiveVersion, domain, &latest_schema, &earliest_opset_where_unchanged);++ assert(latest_schema == nullptr || (latest_schema->SinceVersion() <= maxInclusiveVersion &&+ earliest_opset_where_unchanged == latest_schema->SinceVersion()));++ return latest_schema;+ }++ virtual void GetSchemaAndHistory(+ const std::string& key,+ int maxInclusiveVersion,+ const std::string& domain,+ const ONNX_NAMESPACE::OpSchema** latest_schema,+ int* earliest_opset_where_unchanged) const = 0;+};++/**+@class OnnxRuntimeOpSchemaRegistry++OnnxRuntimeOpSchemaRegistry is used to provide supplement for built-in ONNX schemas.+Each OnnxRuntimeOpSchemaRegistry must register complete opsets delta from a baseline version to max opset version.+(Please notice that baseline opsets are not include in the delta)++For example, ONNXRuntime is build with ONNX 1.2 which is at opset7, to use ONNX opset8 and opset9,+user could create a OnnxRuntimeOpSchemaRegistry and config it as {baseline_opset_version = 7, opset_version = 9}+it means this OnnxRuntimeOpSchemaRegistry contains the complete delta from opset7 to opset9.+*/+class OnnxRuntimeOpSchemaRegistry : public IOnnxRuntimeOpSchemaCollection {+ public:+ OnnxRuntimeOpSchemaRegistry() = default;++ common::Status SetBaselineAndOpsetVersionForDomain(+ const std::string& domain,+ int baseline_opset_version,+ int opset_version);++ DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const override;++ // OnnxRuntimeOpSchemaRegistry must register complete delta for a opset.+ common::Status RegisterOpSet(+ std::vector<ONNX_NAMESPACE::OpSchema>& schemas,+ const std::string& domain,+ int baseline_opset_version,+ int opset_version);++ using IOnnxRuntimeOpSchemaCollection::GetSchema;++ void GetSchemaAndHistory(const std::string& key, int maxInclusiveVersion, const std::string& domain,+ const ONNX_NAMESPACE::OpSchema** latest_schema,+ int* earliest_opset_where_unchanged) const override;++ bool empty() const {+ return map_.empty();+ }++ private:+ common::Status RegisterOpSchema(ONNX_NAMESPACE::OpSchema&& op_schema);++ common::Status RegisterOpSchemaInternal(ONNX_NAMESPACE::OpSchema&& op_schema);++ OrtMutex mutex_;++ OpName_Domain_Version_Schema_Map map_;+ DomainToVersionRangeMap domain_version_range_map_;+};++/**+@class SchemaRegistryManager++SchemaRegistryManager provides a view based on built-in ONNX schema and a list of+OnnxRuntimeOpSchemaRegistry as supplement.++The user needs to make sure the customized schema registry is valid, otherwise the behavior is undefined.++@todo We may add more consistency checks later.+*/+class SchemaRegistryManager : public onnxruntime::IOnnxRuntimeOpSchemaCollection {+ public:+ /**+ Register a new schema registry instance.+ @remarks The schema registry priority is the reverse of registration order. i.e. the last registry added will be+ searched first for a matching OpSchema.+ */+ void RegisterRegistry(std::shared_ptr<IOnnxRuntimeOpSchemaCollection> registry);++ /** Gets the latest opset versions.+ @param is_onnx_only If true, return the latest ONNX schemas. If false, return the latest schemas for all domains.+ */+ DomainToVersionMap GetLatestOpsetVersions(bool is_onnx_only) const override;++ /** Gets the last released opset versions.+ @param is_onnx_only If true, return ONNX schemas only. If false, return the schemas for all domains.+ */+ DomainToVersionMap GetLastReleasedOpsetVersions(bool is_onnx_only) const ;+ /**+ Gets the OpSchema and its history.+ Searches custom schema registries starting with the last one added. \+ If the OpSchema is not found the default ONNX schema registry is searched.++ @param key Operator type.+ @param max_inclusive_version Maximum opset version allowed, inclusive.+ @param domain The domain of the operator.+ @param[out] latest_schema Returns the latest OpSchema if found. nullptr otherwise.+ @param[out] earliest_opset_where_unchanged The earliest opset version preceding max_inclusive_version where the+ operator is known to be unchanged.+ */+ void GetSchemaAndHistory(const std::string& key, int max_inclusive_version, const std::string& domain,+ const ONNX_NAMESPACE::OpSchema** latest_schema,+ int* earliest_opset_where_unchanged) const override;++ private:+ void GetDomainToVersionMapForRegistries(DomainToVersionMap& domain_version_map, bool is_onnx_only) const;++ std::deque<std::shared_ptr<IOnnxRuntimeOpSchemaCollection>> registries;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/optimizer/graph_transformer.h b/include/onnxruntime/core/optimizer/graph_transformer.hnew file mode 100644index 0000000..865a9f3--- /dev/null+++ b/include/onnxruntime/core/optimizer/graph_transformer.h@@ -0,0 +1,72 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#include <string>+#include <unordered_set>++#include "core/common/common.h"+#include "core/graph/graph_viewer.h"+#include "core/optimizer/graph_transformer_level.h"++namespace onnxruntime {++/**+@class GraphTransformer++The interface for in-place transformation of a Graph.+*/+class GraphTransformer {+ public:+ GraphTransformer(const std::string& name, const std::unordered_set<std::string>& compatible_execution_providers = {})+ : name_(name), compatible_provider_types_(compatible_execution_providers) {+ }++ virtual ~GraphTransformer() = default;++ /** Gets the name of this graph transformer. */+ const std::string& Name() const noexcept {+ return name_;+ }++ const std::unordered_set<std::string>& GetCompatibleExecutionProviders() const noexcept {+ return compatible_provider_types_;+ }++ /** Apply the in-place transformation defined by this transformer to the provided Graph instance.+ @param[out] modified Set to true if the Graph was modified.+ @returns Status with success or error information.+ */+ common::Status Apply(Graph& graph, bool& modified, const logging::Logger& logger) const;++ virtual bool ShouldOnlyApplyOnce() const { return false; }++ protected:+ /** Helper method to call ApplyImpl on any subgraphs in the Node. */+ common::Status Recurse(Node& node, bool& modified, int graph_level, const logging::Logger& logger) const {+ int subgraph_level = ++graph_level;+ for (auto& entry : node.GetAttributeNameToMutableSubgraphMap()) {+ auto& subgraph = *entry.second;+ ORT_RETURN_IF_ERROR(ApplyImpl(subgraph, modified, subgraph_level, logger));+ }++ return Status::OK();+ }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphTransformer);++ // Apply the transform to the graph.+ // graph_level is 0 for the main graph, and is incremented when descending into the subgraph of a node.+ // You MUST call Recurse for all valid Nodes in the graph to ensure any subgraphs in control flow nodes+ // (Scan/If/Loop) are processed as well.+ // You should avoid calling Graph::Resolve in ApplyImpl unless you are 100% sure it's required. In most cases+ // the call to Graph::Resolve in Apply prior to ApplyImpl being called, and after ApplyImpl fore the main graph+ // completes (if 'modified' is true) should suffice.+ virtual common::Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger)+ const = 0;++ const std::string name_;+ const std::unordered_set<std::string> compatible_provider_types_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/optimizer/graph_transformer_level.h b/include/onnxruntime/core/optimizer/graph_transformer_level.hnew file mode 100644index 0000000..cc05b21--- /dev/null+++ b/include/onnxruntime/core/optimizer/graph_transformer_level.h@@ -0,0 +1,21 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/common.h"++namespace onnxruntime {++// Graph transformer level+// refer to docs/ONNX_Runtime_Graph_Optimizations.md for details+enum class TransformerLevel : int {+ Default = 0, // required transformers only+ Level1, // basic optimizations+ Level2, // extended optimizations+ Level3, // layout optimizations+ // The max level should always be same as the last level.+ MaxLevel = Level3+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/optimizer/graph_transformer_utils.h b/include/onnxruntime/core/optimizer/graph_transformer_utils.hnew file mode 100644index 0000000..b5d0287--- /dev/null+++ b/include/onnxruntime/core/optimizer/graph_transformer_utils.h@@ -0,0 +1,41 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <gsl/gsl>++#include "core/optimizer/graph_transformer.h"+#include "core/optimizer/rule_based_graph_transformer.h"+#include "core/optimizer/rewrite_rule.h"++namespace onnxruntime {+struct FreeDimensionOverride;+class IExecutionProvider;++namespace optimizer_utils {++/** Generates all predefined rules for this level.+ If rules_to_enable is not empty, it returns the intersection of predefined rules and rules_to_enable.+ TODO: This is visible for testing at the moment, but we should rather make it private. */+std::vector<std::unique_ptr<RewriteRule>> GenerateRewriteRules(TransformerLevel level,+ const std::vector<std::string>& rules_to_enable = {});++/** Generates all predefined (both rule-based and non-rule-based) transformers for this level.+ If transformers_and_rules_to_enable is not empty, it returns the intersection between the predefined transformers/rules+ and the transformers_and_rules_to_enable. */+std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerLevel level,+ gsl::span<const FreeDimensionOverride> free_dimension_overrides,+ const IExecutionProvider& execution_provider /*required by constant folding*/,+ const std::vector<std::string>& rules_and_transformers_to_enable = {});++/** Given a TransformerLevel, this method generates a name for the rule-based graph transformer of that level. */+std::string GenerateRuleBasedTransformerName(TransformerLevel level);++/** Generates all rule-based transformers for this level. */+std::unique_ptr<RuleBasedGraphTransformer> GenerateRuleBasedGraphTransformer(TransformerLevel level,+ const std::vector<std::string>& rules_to_enable,+ const std::unordered_set<std::string>& compatible_execution_providers);++} // namespace optimizer_utils+} // namespace onnxruntimediff --git a/include/onnxruntime/core/optimizer/rewrite_rule.h b/include/onnxruntime/core/optimizer/rewrite_rule.hnew file mode 100644index 0000000..43e7489--- /dev/null+++ b/include/onnxruntime/core/optimizer/rewrite_rule.h@@ -0,0 +1,89 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/common.h"+#include "core/graph/graph_viewer.h"++namespace onnxruntime {++/**+@class RewriteRule++The base class for a rewrite rule. A rewrite rule represents a semantics-preserving transformation of a+computation graph. It can be used to represent, for example, the elimination of operators that serve as+no-ops (e.g., dropout during inference), as well as inlining of "function" definitions or the dual operation+of replacing a complex expression by an equivalent function-call). Unlike the more general GraphTransformer,+a rewrite rule is a more local transformation that is triggered on a particular node of the graph.++Each rule has a set of conditions and a body. The conditions have to be satisfied for the body of the rule+to be triggered. Therefore, when creating a new rewrite rule, two main functions have to be implemented:+- SatisfyCondition defines the condition checks. It is advisable to add the more selective checks first,+ because those will lead to discarding fast rules that cannot be applied on a node.+- Apply is the actual body of the rule that will be executed if SatisfyCondition returns true for a particular+ node. Note that additional, more complex checks can be included in the Apply if putting them in the+ SatisfyCondition would lead to duplicate work (e.g., when we make a check on a Node attribute but we need+ that attribute to execute the rule too).+In general, simple fast checks are a better fit for SatisfyCondition, whereas more complex ones can be added+in the Apply.++In order to avoid evaluating the SatisfyCondition for each rule and each node of the graph, each rewrite rule+should specify the target op types for which a rule will be evaluated, by overriding the TargetOpTypes() function.+If the op type of a node is not included in the target op types of a rule, that rule would not be considered at all.+If the list of op types is left empty, that rule will be triggered for every op type.+*/+class RewriteRule {+ public:+ /**+ @class RewriteRuleEffect++ Class used to indicate the effect of rule application on a graph's node.+ */+ enum class RewriteRuleEffect : uint8_t {+ kNone, // The rewrite rule has not modified the graph.+ kUpdatedCurrentNode, // The rewrite rule updated (but did not remove) the node on which it was triggered.+ kRemovedCurrentNode, // The rewrite rule removed the node on which it was triggered.+ kModifiedRestOfGraph // The rewrite rule modified nodes other than the one it was triggered on.+ };++ RewriteRule(const std::string& name) : name_(name) {}++ virtual ~RewriteRule() = default;++ /** Gets the name of this rewrite rule. */+ const std::string& Name() const noexcept {+ return name_;+ }++ /** Returns the node op types for which this rule will be triggered. If the op type of a node is not included in the+ target op types of a rule, that rule would not be considered at all. Returning an empty list indicates that we+ will attempt to trigger the rule for every op type. */+ virtual std::vector<std::string> TargetOpTypes() const noexcept = 0;++ /** Checks if the condition of the rule is satisfied, and if so applies the body of the rule.+ @param[in] graph The Graph.+ @param[in] node The Node to apply the rewrite to.+ @param[out] rule_effect Enum to indicate if and how the graph was modified as a result of the rule application.+ @returns Status indicating success or providing error information */+ common::Status CheckConditionAndApply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger& logger) const {+ return SatisfyCondition(graph, node, logger) ? Apply(graph, node, rule_effect, logger) : Status::OK();+ }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RewriteRule);++ const std::string name_;++ /** Checks if the Node of the given Graph satisfies the conditions of this rule. The body of the rule will be+ evaluated if this condition function returns true. This can include a more complex pattern matching (conditions+ on the ascending or descending nodes of the node for which this rule was triggered) or some other properties+ of the nodes. */+ virtual bool SatisfyCondition(const Graph& graph, const Node& node, const logging::Logger& logger) const = 0;++ /** This is the actual body of the rule that performs the graph transformation. The transformation happens in-place.+ The return-value of node may be different from the input-value due to rewriting.+ The value of "rule_effect" indicates whether and how the graph was modified by the rule. */+ virtual common::Status Apply(Graph& graph, Node& node, RewriteRuleEffect& rule_effect, const logging::Logger& logger) const = 0;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/optimizer/rule_based_graph_transformer.h b/include/onnxruntime/core/optimizer/rule_based_graph_transformer.hnew file mode 100644index 0000000..04b5072--- /dev/null+++ b/include/onnxruntime/core/optimizer/rule_based_graph_transformer.h@@ -0,0 +1,82 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include "core/common/common.h"+#include "core/graph/graph_viewer.h"+#include "core/optimizer/graph_transformer.h"+#include "core/optimizer/rewrite_rule.h"++namespace onnxruntime {++/**+@class RuleBasedGraphTransformer++Rule-based graph transformer that provides an API to register rewrite rules+and an API to apply all applicable rules to a Graph.++Represents an IGraphTransformer determined by a set of rewrite rules.+The transformer will apply all the rewrite rules iteratively as determined by the underlying rewriting strategy.+Several rewriting-strategies are possible when traversing the graph and applying rewrite rules,+each with different trade offs. At the moment, we define one that performs top-down traversal of nodes.++@TODO: Is a bottom-up traversal more efficient?+@TODO: Is it worth adding the max number of passes a rule should be applied for?+@TODO: We need to define a contract about whether a rewrite rule is allowed to leave+ the graph in an inconsistent state (this will determine when and where we will be+ calling Graph::resolve().+*/+class RuleBasedGraphTransformer : public GraphTransformer {+ public:+ RuleBasedGraphTransformer(const std::string& name,+ const std::unordered_set<std::string>& compatible_execution_providers = {})+ : GraphTransformer(name, compatible_execution_providers) {}++ /** Registers a rewrite rule in this transformer. */+ Status Register(std::unique_ptr<RewriteRule> rule);++ /** Gets the list of registered rewrite rules that will be triggered on nodes with the given op type+ by this rule-based transformer.+ @returns a pointer to the vector containing all the registered rewrite rules. */+ const std::vector<std::reference_wrapper<const RewriteRule>>* GetRewriteRulesForOpType(const std::string& op_type) const {+ auto rules = op_type_to_rules_.find(op_type);+ return (rules != op_type_to_rules_.cend()) ? &rules->second : nullptr;+ }++ /** Gets the rewrite rules that are evaluated on all nodes irrespective of their op type.+ @returns a pointer to the vector containing all such rewrite rules or nullptr if no such rule. */+ const std::vector<std::reference_wrapper<const RewriteRule>>* GetAnyOpRewriteRules() const {+ return &any_op_type_rules_;+ }++ /** Returns the total number of rules that are registered in this transformer. */+ size_t RulesCount() const;++ protected:+ /** Applies the given set of rewrite rules on the Node of this Graph.+ @param[in] graph The Graph.+ @param[in] node The Node to apply the rules to.+ @param[in] rules The vector of RewriteRules that will be applied to the Node.+ @param[out] rule_effect Enum that indicates whether and how the graph was modified as a result of+ applying rules on this node.+ @returns Status indicating success or providing error information. */+ common::Status ApplyRulesOnNode(Graph& graph, Node& node,+ const std::vector<std::reference_wrapper<const RewriteRule>>& rules,+ RewriteRule::RewriteRuleEffect& rule_effect, const logging::Logger& logger) const;++ private:+ using RuleEffect = RewriteRule::RewriteRuleEffect;++ // The list of unique pointers for all rules (so that rules can be registered for several op types).+ std::vector<std::unique_ptr<RewriteRule>> rules_;+ // Map that associates a node's op type with the vector of rules that are registered to be triggered for that node.+ std::unordered_map<std::string, std::vector<std::reference_wrapper<const RewriteRule>>> op_type_to_rules_;+ // Rules that will be evaluated regardless of the op type of the node.+ std::vector<std::reference_wrapper<const RewriteRule>> any_op_type_rules_;++ // Performs a single top-down traversal of the graph and applies all registered rules.+ common::Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;+};++} // namespace onnxruntimediff --git a/include/onnxruntime/core/platform/Barrier.h b/include/onnxruntime/core/platform/Barrier.hnew file mode 100644index 0000000..915cfc5--- /dev/null+++ b/include/onnxruntime/core/platform/Barrier.h@@ -0,0 +1,81 @@+// This file is part of Eigen, a lightweight C++ template library+// for linear algebra.+//+// Copyright (C) 2016 Dmitry Vyukov <[email protected]>+//+// This Source Code Form is subject to the terms of the Mozilla+// Public License v. 2.0. If a copy of the MPL was not distributed+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.++#include <assert.h>++#include "core/common/spin_pause.h"+#include "core/platform/ort_mutex.h"++#include <mutex>+#include <atomic>++namespace onnxruntime {+class Barrier {+ public:+ explicit Barrier(unsigned int count, bool spin = false)+ : state_(count << 1), notified_(false), spin_(spin) {+ assert(((count << 1) >> 1) == count);+ }+#ifdef NDEBUG+ ~Barrier() = default;+#else+ ~Barrier() {+ assert((state_ >> 1) == 0);+ }+#endif++ void Notify(unsigned int c = 1) {+ unsigned int delta = c << 1;+ unsigned int v = state_.fetch_sub(delta, std::memory_order_acq_rel) - delta;+ if (v != 1) {+ // Clear the lowest bit (waiter flag) and check that the original state+ // value was not zero. If it was zero, it means that notify was called+ // more times than the original count.+ assert(((v + delta) & ~1) != 0);+ return; // either count has not dropped to 0, or waiter is not waiting+ }+ std::unique_lock<OrtMutex> l(mu_);+ assert(!notified_);+ notified_ = true;+ cv_.notify_all();+ }++ void Wait() {+ if (spin_) {+ while ((state_ >> 1) != 0) {+ onnxruntime::concurrency::SpinPause();+ }+ } else {+ unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel);+ if ((v >> 1) == 0)+ return;+ std::unique_lock<OrtMutex> l(mu_);+ while (!notified_) {+ cv_.wait(l);+ }+ }+ }++ private:+ OrtMutex mu_;+ OrtCondVar cv_;+ std::atomic<unsigned int> state_; // low bit is waiter flag+ bool notified_;+ const bool spin_;+};++// Notification is an object that allows a user to to wait for another+// thread to signal a notification that an event has occurred.+//+// Multiple threads can wait on the same Notification object,+// but only one caller must call Notify() on the object.+struct Notification : Barrier {+ Notification() : Barrier(1){};+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.hnew file mode 100644index 0000000..f24a114--- /dev/null+++ b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h@@ -0,0 +1,1391 @@+// This file is part of Eigen, a lightweight C++ template library+// for linear algebra.+//+// Copyright (C) 2016 Dmitry Vyukov <[email protected]>+//+// This Source Code Form is subject to the terms of the Mozilla+// Public License v. 2.0. If a copy of the MPL was not distributed+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.++/* Modifications Copyright (c) Microsoft. */++#include <type_traits>++#pragma once+#include "onnxruntime_config.h"+// build/external/eigen/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h:162:71:+// error: ignoring attributes on template argument "Eigen::PacketType<const float, Eigen::DefaultDevice>::type {aka+// __vector(4) float}" [-Werror=ignored-attributes]+#if defined(__GNUC__)+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wunused-parameter"+#elif defined(_MSC_VER)+#pragma warning(push)+#pragma warning(disable : 4127)+#pragma warning(disable : 4805)+#endif++#include "unsupported/Eigen/CXX11/ThreadPool"++#if defined(__GNUC__)+#pragma GCC diagnostic pop+#elif defined(_MSC_VER)+#pragma warning(pop)+#endif+#include "core/common/denormal.h"+#include "core/common/make_unique.h"+#include "core/common/spin_pause.h"+#include "core/platform/ort_mutex.h"+#include "core/platform/Barrier.h"++// ORT thread pool overview+// ------------------------+//+// The ORT thread pool implementation is split into two layers. This+// file provides the low-level component. See the accompanying+// comments in threadpool.h for the high-level component.+//+// The code here is derived from the Eigen non-blocking thread pool,+// although many parts have been updated over time. The main+// abstractions used here are:+//+// - The thread pool maintains a set of OS threads running+// ThreadPoolTempl::WorkerLoop.+//+// Each thread has its own RunQueue object, holding a queue of tasks+// that have been pushed to the thread for execution. The main work+// loop is to pop a task from the head of the queue, and to execute+// it to completion. If the worker's run queue is empty then it+// will spin waiting for work, then attempt to steal tasks from+// other threads' queues, and then block in the OS if it cannot find+// work.+//+// This spin-then-block behavior is configured via a flag provided+// when creating the thread pool, and by the constant spin_count.+//+// - Although all tasks are simple void()->void functions,+// conceptually there are three different kinds:+//+// - One-shot tasks submitted externally via the Schedule() method.+// These tasks are used to support asynchronous work. These are+// used in the parallel executor, but otherwise are not widely+// used outside of test harnesses (see threadpool_test.cc for some+// examples).+//+// - Tasks for running a parallel loop.+//+// The tasks themselves are defined in threadpool.cc, and are+// submitted to the run queues via RunInParallel->SummonWorkers.+// Each task will loop internally, picking off iterations from the+// user's code via atoic-fetch-and-add, until the loop is+// complete.+//+// This two-layer approach lets us separate out the+// super-lightweight per-iteration-batch work from the more+// costsly per-loop work of managing Task objects.+//+// - Tasks for running a parallel section. This is an extension of+// the approach taken for parallel loops. However, the Tasks are+// defined in this file, and can pick up iterations from a series+// of different parallel loops. The tasks are defined in+// RunInParallelSection->SummonWorkers.+//+// The additional layer of parallel sections is a further way to+// amortize costs: the work done creating the tasks can be+// performed once, and then exploited over a series of loops.+//+// There are a few aspects of the modified Eigen thread pool to+// highlight:+//+// - The run queues follow the usual approach of having push/pop+// operations on the front/back, and optimizing the PopFront case+// for single-threaded use by the thread owning the run queue.+//+// However, we support an additional Revoke operation to replace an+// item in the middle of a queue with a tombstone. This operation+// is used at the end of parallel loops and parallel sections to+// remove any tasks that were created but not yet executed. Once+// revoked, a thread can rely on the fact that the task will no+// longer execute. Revocation helps manage captured state in+// parallel loops: the alternatives would be (i) waiting for all+// tasks that captured state to reach the head of their queues and+// execute, or (ii) use heap-allocated state in tasks, and use a+// technique such as reference counting to de-allocate it.+//+// To support revoation, each thread has a unique "Tag" to identify+// the items that it adds to the work queues. A thread can revoke+// an item only if it has the thread's own tag.+//+// - The worker threads maintain a best-effort bitmap in+// good_worker_hints_ of which threads to push work to. A thread+// controls its status via SetGoodWorkerHint. A thread is a "good"+// worker when it is actively spinning for work, meaning both that+// it is not blocked in the OS, and that it is not busy with work+// already.+//+// This heuristic aims to avoid waking additional sleeping threads+// where possible, and in a series of parallel loops or parallel+// sections to push the work to the same set of threads each time.++namespace onnxruntime {+namespace concurrency {++class ThreadPoolParallelSection;+class ThreadPoolLoop;++// Align to avoid false sharing with prior fields. If required,+// alignment or padding must be added subsequently to avoid false+// sharing with later fields. Note that:+//+// - The __x86_64__ value is twice the line size (64 bytes). This+// accounts for 2-line prefetch behavior on some cores.+//+// - Ideally, ORT_ALIGN_TO_AVOID_FALSE_SHARING is used. However, the+// definition of ThreadPoolParallelSection uses naive padding+// because C++11 does not support alignment constraints on+// allocation or expose stdlib.h aligned_alloc. C++17 introduces+// support for aligned allocation which we could use here.++#if defined(__x86_64__)+#define ORT_FALSE_SHARING_BYTES 128+#else+#define ORT_FALSE_SHARING_BYTES 64+#endif++#define ORT_ALIGN_TO_AVOID_FALSE_SHARING alignas(ORT_FALSE_SHARING_BYTES)++struct PaddingToAvoidFalseSharing {+ char padding[ORT_FALSE_SHARING_BYTES];+};++// Extended Eigen thread pool interface, avoiding the need to modify+// the ThreadPoolInterface.h header from the external Eigen+// repository.++class ExtendedThreadPoolInterface : public Eigen::ThreadPoolInterface {+ public:+ // Start/end a parallel section, within which calls to+ // RunInParallelSection may be made. Parallel sections are+ // non-nesting.+ virtual std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)> AllocateParallelSection() = 0;+ virtual void StartParallelSection(ThreadPoolParallelSection &ps) = 0;+ virtual void EndParallelSection(ThreadPoolParallelSection &ps) = 0;++ // Run fn with up to n degree-of-parallelism enlisting the thread+ // pool for help. The degree-of-parallelism includes the caller,+ // and so if n==1 then the function will run directly in the caller.+ //+ // The fork-join synchronization is handled in the thread pool, and+ // so any state captured by fn() is safe from concurrent access once+ // RunInParallelSection returns.+ //+ // The parameter idx provides a loop-local thread ID in the range+ // [0,k) where k<=n.+ virtual void RunInParallelSection(ThreadPoolParallelSection &ps,+ std::function<void(unsigned idx)> fn,+ unsigned n) = 0;++ // Special case alternative to RunInParallelSection for use without+ // an existing parallel section. Ideally we would use a single+ // iplemenation and a stack-allocated ThreadPoolParallelSection.+ //+ // However, on the BM_ThreadPoolParallelFor microbenchmark I saw+ // ~20% overhead on the resulting single-loop parallel sections.+ // There are some additional costs (~5%) for additional invocations+ // through lambda functions on loop entry. Most significantly, on+ // loop exit, we incurred ~15% cost by no longer being able to+ // overlap clean-up of unused Task objects in EndParallelSection+ // with waiting for loop iterations to complete.+ //+ // [ Note that this 20% overhead is more than paid for when we have+ // two loops execute in series in a parallel section. ]+ virtual void RunInParallel(std::function<void(unsigned idx)> fn,+ unsigned n) = 0;+};+++class ThreadPoolParallelSection {+ public:+ // State accessed only by the main thread+ // --------------------------------------++ // Tasks successfully submitted to the work queues. This sets the+ // maximum degree of parallelism that the section will support.+ std::vector<std::pair<int,unsigned>> tasks;++ // State shared between the main thread and worker threads+ // -------------------------------------------------------++ // Flag to signal termination of the parallel section+ std::atomic<bool> active{false};++ std::atomic<unsigned> worker_idx{0};++ // Count of the number of tasks that completed normally. Other+ // tasks may be running currently, or may be present in work queues,+ // or may have been removed from the queues by+ // RunQueue::RevokeWithTag.+ PaddingToAvoidFalseSharing padding_1;+ std::atomic<unsigned> tasks_finished{0};+ PaddingToAvoidFalseSharing padding_2;++ // If non-null, the current loop that tasks should be executing. We+ // need to be careful on access to the contents of current_loop+ // because it can be stack allocated on the thread entering the+ // loop:+ //+ // - Readers increment workers_in_loop and then read current_loop+ //+ // - Writers wishing to deallocate *current_loop must first clear+ // current_loop and then wait for workers_in_loop==0+ std::atomic<ThreadPoolLoop *> current_loop{nullptr};+ std::atomic<unsigned> workers_in_loop{0};+};++class ThreadPoolLoop {+ public:+ ThreadPoolLoop(std::function<void(unsigned)> f, unsigned t) : fn(std::move(f)), threads_needed(t) {+ }++ const std::function<void(unsigned)> fn;+ const unsigned threads_needed;++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ThreadPoolLoop);+};++template <typename Work, typename Tag, unsigned kSize>+class RunQueue {+ public:+ RunQueue() : front_(0), back_(0) {+ // require power-of-two for fast masking+ assert((kSize & (kSize - 1)) == 0);+ assert(kSize > 2); // why would you do this?+ assert(kSize <= (64 << 10)); // leave enough space for counter+ for (unsigned i = 0; i < kSize; i++) array_[i].state.store(ElemState::kEmpty, std::memory_order_relaxed);+ }++ ~RunQueue() {+ assert(Size() == 0);+ }++ // PushFront inserts w at the beginning of the queue.+ // If queue is full returns w, otherwise returns default-constructed Work.+ Work PushFront(Work w) {+ unsigned front = front_.load(std::memory_order_relaxed);+ Elem& e = array_[front & kMask];+ ElemState s = e.state.load(std::memory_order_relaxed);+ if (s != ElemState::kEmpty ||+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))+ return w;+ front_.store(front + 1 + (kSize << 1), std::memory_order_relaxed);+ e.w = std::move(w);+ e.tag = Tag();+ e.state.store(ElemState::kReady, std::memory_order_release);+ return Work();+ }++ // PopFront removes and returns the first element in the queue.+ // If the queue was empty returns default-constructed Work.+ Work PopFront() {+ unsigned front;+ Elem *e;+ ElemState s;++ // Drain revoked items from the front of the queue. CAS to busy to synchronize with+ // any attempt to take the same item from the back of the queue.+ do {+ front = front_.load(std::memory_order_relaxed);+ e = &array_[(front - 1) & kMask];+ s = e->state.load(std::memory_order_relaxed);+ if (s == ElemState::kRevoked &&+ e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {+ e->state.store(ElemState::kEmpty, std::memory_order_release);+ front = ((front - 1) & kMask2) | (front & ~kMask2);+ front_.store(front, std::memory_order_relaxed);+ }+ } while (s == ElemState::kRevoked);++ // Attempt to take next item. State kEmpty shows the queue is empty, kBusy shows+ // the work is in progress on the item at the front of the queue.+ if (s != ElemState::kReady ||+ !e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))+ return Work();+ Work w = std::move(e->w);+ e->tag = Tag();+ e->state.store(ElemState::kEmpty, std::memory_order_release);+ front = ((front - 1) & kMask2) | (front & ~kMask2);+ front_.store(front, std::memory_order_relaxed);+ return w;+ }++ // PushBack adds w at the end of the queue.+ // If queue is full returns w, otherwise returns default-constructed Work.+ Work PushBack(Work w) {+ std::unique_lock<OrtMutex> lock(mutex_);+ unsigned back = back_.load(std::memory_order_relaxed);+ Elem& e = array_[(back - 1) & kMask];+ ElemState s = e.state.load(std::memory_order_relaxed);+ if (s != ElemState::kEmpty ||+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))+ return w;+ back = ((back - 1) & kMask2) | (back & ~kMask2);+ back_.store(back, std::memory_order_relaxed);+ e.w = std::move(w);+ e.tag = Tag();+ e.state.store(ElemState::kReady, std::memory_order_release);+ return Work();+ }++ // PushBackWithTag adds w at the end of the queue. The tag value can be used on a+ // subsequent call to RevokeWithTag to remove the item from the queue in combination+ // with w_idx. Typically the tag will be a per-thread ID to distinguish work+ // submitted from different threads.+ //+ // If the queue is full, returns w, otherwise returns default-constructed work.+ Work PushBackWithTag(Work w, Tag tag, unsigned &w_idx) {+ std::unique_lock<OrtMutex> lock(mutex_);+ unsigned back = back_.load(std::memory_order_relaxed);+ w_idx = (back-1) & kMask;+ Elem& e = array_[w_idx];+ ElemState s = e.state.load(std::memory_order_relaxed);+ if (s != ElemState::kEmpty ||+ !e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))+ return w;+ back = ((back - 1) & kMask2) | (back & ~kMask2);+ back_.store(back, std::memory_order_relaxed);+ e.w = std::move(w);+ e.tag = tag;+ e.state.store(ElemState::kReady, std::memory_order_release);+ return Work();+ }++ // PopBack removes and returns the last elements in the queue.+ Work PopBack() {+ if (Empty())+ return Work();+ std::unique_lock<OrtMutex> lock(mutex_);+ unsigned back;+ Elem *e;+ ElemState s;++ // Drain revoked items from the back of the queue. CAS to busy to synchronize with+ // any attempt to take the same item from the front of the queue.+ do {+ back = back_.load(std::memory_order_relaxed);+ e = &array_[back & kMask];+ s = e->state.load(std::memory_order_relaxed);+ if (s == ElemState::kRevoked &&+ e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {+ e->state.store(ElemState::kEmpty, std::memory_order_release);+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);+ }+ } while (s == ElemState::kRevoked);++ if (s != ElemState::kReady ||+ !e->state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire))+ return Work();+ Work w = std::move(e->w);+ e->tag = Tag();+ e->state.store(ElemState::kEmpty, std::memory_order_release);+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);+ return w;+ }++ // RevokeItem removes a work item from the queue. Items are identified positionally,+ // and so a tag is used to detect whether the same position is occupied by a+ // different work item at the time of removal. RevokeWithTags lets threads offer work+ // for parallel execution, and then revoke the offer prior to the work executing (for+ // instance if the thread itself completes all of the work). Revoking the work+ // lets the thread deallocate state that might otherwise have been captured by the work item+ // and accessed by it.+ //+ // Return true iff the item is successfully revoked. If the item is not revoked then+ // the caller must assume that it may still execute, for instance because it+ // has been pop'd from the queue concurrent with the revocation request.++ bool RevokeWithTag(Tag tag, unsigned w_idx) {+ bool revoked = false;+ std::unique_lock<OrtMutex> lock(mutex_);+ Elem& e = array_[w_idx];+ ElemState s = e.state.load(std::memory_order_relaxed);++ // We have acquired a lock on the queue, synchronizing with+ // operations aside from the PopFront fast-path. Synchronize with+ // that by attempting the same kReady->kBusy transition via CAS.++ if (s == ElemState::kReady &&+ e.state.compare_exchange_strong(s, ElemState::kBusy, std::memory_order_acquire)) {+ if (e.tag == tag) {+ unsigned back = back_.load(std::memory_order_relaxed);+ unsigned back_idx = back & kMask;+ if (back_idx != w_idx) {+ // Item is not at the back of the queue, mark it in-place as revoked+ e.tag = Tag();+ e.w = Work();+ e.state.store(ElemState::kRevoked, std::memory_order_release);+ revoked = true;+ } else {+ // Item being removed as still at the back; shift the back pointer over it,+ // and bump the version number.+ e.tag = Tag();+ e.w = Work();+ e.state.store(ElemState::kEmpty, std::memory_order_release);+ back_.store(back + 1 + (kSize << 1), std::memory_order_relaxed);+ revoked = true;+ }+ } else {+ // Tag mismatch, i.e. work queue slot re-used+ e.state.store(ElemState::kReady, std::memory_order_release);+ }+ }+ return revoked;+ }++ // Size returns current queue size.+ // Can be called by any thread at any time.+ unsigned Size() const {+ return SizeOrNotEmpty<true>();+ }++ // Empty tests whether container is empty.+ // Can be called by any thread at any time.+ bool Empty() const {+ return SizeOrNotEmpty<false>() == 0;+ }++ // Delete all the elements from the queue.+ void Flush() {+ while (!Empty()) {+ PopFront();+ }+ }++ private:+ static const unsigned kMask = kSize - 1;+ static const unsigned kMask2 = (kSize << 1) - 1;++ enum class ElemState : uint8_t {+ kEmpty,+ kBusy,+ kReady,+ kRevoked,+ };++ // Updates to an element are bracketed by a std::memory_order_acquire+ // load from the state, and a std::memory_order_release store. Accesses+ // to the front/back indices for the work queue use relaxed semantics,+ // with the state of the elements being authoritative.+ //+ // TODO: Revisit whether there is a significant benefit for the current+ // workloads in the complexity here.+ struct Elem {+ std::atomic<ElemState> state;+ Tag tag;+ Work w;+ };++ OrtMutex mutex_;++ // Low log(kSize) + 1 bits in front_ and back_ contain rolling index of+ // front/back, respectively. The remaining bits contain modification counters+ // that are incremented on Push operations. This allows us to (1) distinguish+ // between empty and full conditions (if we would use log(kSize) bits for+ // position, these conditions would be indistinguishable); (2) obtain+ // consistent snapshot of front_/back_ for Size operation using the+ // modification counters.+ ORT_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> front_;+ ORT_ALIGN_TO_AVOID_FALSE_SHARING std::atomic<unsigned> back_;+ ORT_ALIGN_TO_AVOID_FALSE_SHARING Elem array_[kSize];++ // SizeOrNotEmpty returns current queue size; if NeedSizeEstimate is false,+ // only whether the size is 0 is guaranteed to be correct.+ // Can be called by any thread at any time.+ template <bool NeedSizeEstimate>+ unsigned SizeOrNotEmpty() const {+ // Emptiness plays critical role in thread pool blocking. So we go to great+ // effort to not produce false positives (claim non-empty queue as empty).+ unsigned front = front_.load(std::memory_order_acquire);+ for (;;) {+ // Capture a consistent snapshot of front/tail.+ unsigned back = back_.load(std::memory_order_acquire);+ unsigned front1 = front_.load(std::memory_order_relaxed);+ if (front != front1) {+ front = front1;+ std::atomic_thread_fence(std::memory_order_acquire);+ continue;+ }+ if (NeedSizeEstimate) {+ return CalculateSize(front, back);+ }+ // This value will be 0 if the queue is empty, and undefined otherwise.+ unsigned maybe_zero = ((front ^ back) & kMask2);+ // Queue size estimate must agree with maybe zero check on the queue+ // empty/non-empty state.+ eigen_assert((CalculateSize(front, back) == 0) == (maybe_zero == 0));+ return maybe_zero;+ }+ }++ EIGEN_ALWAYS_INLINE+ unsigned CalculateSize(unsigned front, unsigned back) const {+ int size = (front & kMask2) - (back & kMask2);+ // Fix overflow.+ if (size < 0)+ size += 2 * kSize;+ // Order of modification in push/pop is crafted to make the queue look+ // larger than it is during concurrent modifications. E.g. push can+ // increment size before the corresponding pop has decremented it.+ // So the computed size can be up to kSize + 1, fix it.+ if (size > static_cast<int>(kSize))+ size = kSize;+ return static_cast<unsigned>(size);+ }++ RunQueue(const RunQueue&) = delete;+ void operator=(const RunQueue&) = delete;+};++static std::atomic<uint32_t> next_tag{1};++template <typename Environment>+class ThreadPoolTempl : public onnxruntime::concurrency::ExtendedThreadPoolInterface {++ private:+ struct PerThread;++ static unsigned WorkerLoop(int id, Eigen::ThreadPoolInterface* param) {+ // unsafe downcast+ ThreadPoolTempl* this_ptr = (ThreadPoolTempl*)param;+ this_ptr->WorkerLoop(id);+ return 0;+ }++ public:+ struct Tag {+ constexpr Tag() : v_(0) {+ }++ Tag(uint32_t v) : v_(v) {+ }++ // Allocate a new tag to use to identify work items from a given+ // thread in a parallel section. Ideally, threads will have+ // unique tags, but re-use is not incorrect if the counter wraps+ // (for intsance, if a long-running workload is calling into ORT+ // from a fresh thread for each request). We must not re-use the+ // default tag 0 which is used to identify work items added via+ // Schedule as opposed to requests for help in parallel sections.++ static Tag GetNext() {+ Tag t = Tag(next_tag++);+ if (t.v_ == 0) {+ t = Tag(next_tag++);+ }+ return t;+ }++ uint32_t Get() const {+ return v_;+ }++ bool operator==(Tag& other) const {+ return v_ == other.v_;+ }++ uint32_t v_ = 0;+ };++ typedef std::function<void()> Task;+ typedef RunQueue<Task, Tag, 1024> Queue;+#ifdef _WIN32+ using CHAR_TYPE = wchar_t;+#else+ using CHAR_TYPE = char;+#endif+ ThreadPoolTempl(const CHAR_TYPE* name, int num_threads, bool allow_spinning, Environment& env,+ const ThreadOptions& thread_options)+ : env_(env),+ num_threads_(num_threads),+ allow_spinning_(allow_spinning),+ set_denormal_as_zero_(thread_options.set_denormal_as_zero),+ worker_data_(num_threads),+ all_coprimes_(num_threads),+ blocked_(0),+ done_(false),+ cancelled_(false) {++ // Calculate coprimes of all numbers [1, num_threads].+ // Coprimes are used for random walks over all threads in Steal+ // and NonEmptyQueueIndex. Iteration is based on the fact that if we take+ // a random starting thread index t and calculate num_threads - 1 subsequent+ // indices as (t + coprime) % num_threads, we will cover all threads without+ // repetitions (effectively getting a presudo-random permutation of thread+ // indices).+ for (int i = 1; i <= num_threads_; ++i) {+ all_coprimes_.emplace_back(i);+ ComputeCoprimes(i, &all_coprimes_.back());+ }++ // Allocate space for per-thread bits to indicate which threads to consider+ // preferable for pushing work. We use a regular array given that a std::vector+ // cannot contain std::atomic.+ num_hint_words_ = static_cast<int>((num_threads_ + bits_per_hint_word_ - 1) / bits_per_hint_word_);+ good_worker_hints_ = onnxruntime::make_unique<std::atomic<uint64_t>[]>(num_hint_words_);++ worker_data_.resize(num_threads_);+ for (int i = 0; i < num_threads_; i++) {+ worker_data_[i].thread.reset(env_.CreateThread(name, i, WorkerLoop, this, thread_options));+ }+ }++ ~ThreadPoolTempl() override {+ done_ = true;++ // Now if all threads block without work, they will start exiting.+ // But note that threads can continue to work arbitrary long,+ // block, submit new work, unblock and otherwise live full life.+ if (!cancelled_) {+ WakeAllWorkersForExit();+ } else {+ // Since we were cancelled, there might be entries in the queues.+ // Empty them to prevent their destructor from asserting.+ for (size_t i = 0; i < worker_data_.size(); i++) {+ worker_data_[i].queue.Flush();+ }+ }+ // Join threads explicitly (by destroying) to avoid destruction order within+ // this class.+ for (size_t i = 0; i < worker_data_.size(); ++i) worker_data_[i].thread.reset();+ }++ // Run fn(). Ordinarily, the function will be added to the thread pool and executed+ // by a worker thread. If the thread pool rejects the work then fn() will instead+ // execute synchronously during Schedule(fn). Currently the thread pool will only+ // reject work if the queue of pending work is full.++ void Schedule(std::function<void()> fn) override {+ PerThread* pt = GetPerThread();+ if (pt->pool == this) {+ // Worker thread of this pool, push onto the thread's queue.+ Queue& q = worker_data_[pt->thread_id].queue;+ fn = q.PushFront(std::move(fn));+ } else {+ // A free-standing thread (or worker of another pool), push onto a random+ // queue.+ int q_idx = Rand(&pt->rand) % num_threads_;+ WorkerData &td = worker_data_[q_idx];+ Queue& q = td.queue;+ fn = q.PushBack(std::move(fn));+ if (!fn) {+ // The queue accepted the work; ensure that the thread will pick it up+ td.EnsureAwake();+ }+ }++ // Run the work directly if the queue rejected the work+ if (fn) fn();+ }++// The thread pool maintains a set of hints for which threads will be good to distribute+// work to. A thread is considered "good" if it is actively spinning, meaning both that+// it is not busy with existing work, and that it should respond quickly to the addition+// of new work.++void SetGoodWorkerHint(int idx, bool is_good) {+ assert(idx >= 0 && idx < num_threads_);+ std::atomic<uint64_t>& u64 = good_worker_hints_[idx / bits_per_hint_word_];+ uint64_t bit = 1ull << (idx % bits_per_hint_word_);+ uint64_t saw, want;+ do {+ saw = u64.load();+ want = is_good ? (saw|bit) : (saw&~bit);+ } while (!u64.compare_exchange_weak(saw, want));+}++// Retrieve hints for up to n threads to distribute work to. Threads in good_hints+// pass a best-effort check to identify spinning threads via the good_worker_hints_+// bitmap. Threads in alt_hint do not pass that test, but are distinct from those in+// good_hints, letting the caller avoid distributing more than one work item to+// any individual thread.++void GetGoodWorkerHints(unsigned n, std::vector<unsigned>& good_hints, std::vector<unsigned>& alt_hints) {+ PerThread* pt = GetPerThread();+ unsigned need_alt = n;+ good_hints.clear();+ alt_hints.clear();++ // Iterate through the words of hints, starting from a pseudo-randomly chosen+ // base. This aims to distribute work across large machines in cases we+ // have multiple threads scheduling work concurrently.++ unsigned base = Rand(&pt->rand) % num_hint_words_;+ for (unsigned i = 0u; n && (i < num_hint_words_); i++) {+ int u64_idx = (base + i) % num_hint_words_;+ std::atomic<uint64_t>* u64 = &good_worker_hints_[u64_idx];+ uint64_t saw = u64->load();+ uint64_t want = saw;++ // Pick up to n bits that are set in the current word+ for (unsigned j = 0u; n && (j < bits_per_hint_word_); j++) {+ uint64_t bit = 1ull << j;+ int thread = u64_idx * bits_per_hint_word_ + j;+ if (saw & bit) {+ good_hints.push_back(thread);+ want &= ~bit;+ n--;+ } else if (need_alt && thread < num_threads_) {+ alt_hints.push_back(thread);+ need_alt--;+ }+ }++ // Best-effort attempt to remove the hints. We should measure the impact of+ // contention here, but the intuition is that if we conflict on the CAS then the+ // machine is likely to be busy in any case, and we will have queuing on the+ // work items.+ u64->compare_exchange_strong(saw, want);+ }+}++//......................................................................+//+// Parallel sections+// -----------------+//+// Allocate a new ThreadPoolParallelSection, owned by the returned+// unique_ptr. The explicit deleter avoids the Eigen-specific+// definition of ThreadPoolParallelSection needing to be avilable in+// threadpool.h where the user-facing parallel section API is defined.++std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)> AllocateParallelSection() override {+ return std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)>+ (new ThreadPoolParallelSection,+ [](ThreadPoolParallelSection *tps) {+ delete tps;+ });+}++// Start a parallel section, using a caller-provided+// ThreadPoolParallelSection for maintaining the per-section state.+// Starting a parallel section is just book-keeping; threads are+// "summoned" to help with the parallel section once it enters+// parallel loops. The threads are then retained until the end of the+// section, being re-used over subsequent loops.++void StartParallelSectionInternal(PerThread &pt,+ ThreadPoolParallelSection &ps) {+ assert((!pt.leading_par_section) && "Nested parallelism not supported");+ assert((!ps.active) && "Starting parallel section, but active already");+ pt.leading_par_section = true;+ if (!pt.tag.Get()) {+ pt.tag = Tag::GetNext();+ }+ ps.active = true;+}++void StartParallelSection(ThreadPoolParallelSection &ps) override {+ PerThread* pt = GetPerThread();+ StartParallelSectionInternal(*pt, ps);+}++// End a parallel section, waiting for all worker threads to exit from+// section. Hence, on return, the ThreadPoolParallelSection object+// can be dealloacted.++void EndParallelSectionInternal(PerThread &pt,+ ThreadPoolParallelSection &ps) {+ assert((pt.leading_par_section) && "Ending parallel section, but none started");+ assert((ps.active) && "Ending parallel section, but not active");+ pt.leading_par_section = false;++ // Notify workers to exit from the section+ ps.active = false;++ // Attempt to revoke any tasks that were sent to workers but not+ // started.+ unsigned tasks_started = static_cast<unsigned>(ps.tasks.size());+ unsigned tasks_revoked = 0;+ while (!ps.tasks.empty()) {+ const auto& item = ps.tasks.back();+ Queue& q = worker_data_[item.first].queue;+ if (q.RevokeWithTag(pt.tag, item.second)) {+ tasks_revoked++;+ }+ ps.tasks.pop_back();+ }++ // Wait for workers to exit ParLoopWorker+ auto tasks_to_wait_for = tasks_started - tasks_revoked;+ while (ps.tasks_finished < tasks_to_wait_for) {+ onnxruntime::concurrency::SpinPause();+ }++ // Clear status to allow the ThreadPoolParallelSection to be+ // re-used.+ ps.tasks_finished = 0;+}++void EndParallelSection(ThreadPoolParallelSection &ps) override {+ PerThread* pt = GetPerThread();+ EndParallelSectionInternal(*pt, ps);+}++//......................................................................+//+// Parallel loops+// --------------+//+// Ensure that the ThreadPoolParallelSection has sufficient workers to+// execute a loop with degree of parallelism n. We track the number+// of workers already avaiable to the parallel section, prior to+// submitting tasks to the work queues to make up the total.+//+// Each worker will call in to worker_fn(idx) with a per-worker thread+// ID. Note there are different levels of indirection here:+//+// - In a single-loop parallel section, worker_fn will directly+// execute the threadpool.cc code that implements the parallel loop.+//+// - In a multi-loop parallel section, worker_fn is an intermediate+// function that is long-lived (i.e., that lasts until the end of+// the parallel section, as opposed to just a single loop's+// duration).++void SummonWorkers(PerThread &pt,+ ThreadPoolParallelSection &ps,+ unsigned n,+ const std::function<void(unsigned)> &worker_fn) {+ // Wrap the user's worker function with one that allocates a unique+ // worker index for the loop, and synchronizes (as the last step)+ // with the exit path in EndParallelSection. In principle we could+ // allocate worker IDs during the loop below and capture them by+ // value. However, the costs of creating distinct lambda for each+ // iteration appeared more costly than the cost of synchronization+ // on a shared counter.+ auto call_worker_fn = [&ps, worker_fn]() {+ unsigned my_idx = ++ps.worker_idx;+ worker_fn(my_idx);+ // After the assignment to ps.tasks_finished, the stack-allocated+ // ThreadPoolParallelSection object may be destroyed.+ ps.tasks_finished++;+ };++ // Identify whether we need to create additional workers.+ // Throughout the threadpool implementation, degrees of parallelism+ // ("n" here) refer to the total parallelism including the main+ // thread. Hence we consider the number of existing tasks + 1.+ unsigned current_dop = static_cast<unsigned>(ps.tasks.size()) + 1;+ if (n > current_dop) {+ unsigned extra_needed = n - current_dop;++ // Obtain hints for which worker threads to push the tasks to.+ // This uses a best-effort assessment of which threads are+ // spinning.+ std::vector<unsigned> good_hints, alt_hints;+ GetGoodWorkerHints(extra_needed, good_hints, alt_hints);++ // Create the additional tasks, and push them to workers.+ for (auto i = 0u; i < extra_needed; i++) {+ Task t;+ int q_idx;+ if (i < good_hints.size()) {+ q_idx = good_hints[i];+ } else {+ auto alt_i = i - static_cast<unsigned>(good_hints.size());+ if (alt_i < alt_hints.size()) {+ q_idx = alt_hints[alt_i];+ } else {+ q_idx = Rand(&pt.rand) % num_threads_;+ }+ }++ // If the worker's queue accepts the task, then record it in+ // the vector of tasks that we will need to synchronize with on+ // exiting the parallel section. If the queue rejects the task+ // (perhaps because it is full) then we take no further action:+ // in a parallel loop we will always be running work on the+ // main thread, providing progress.+ WorkerData& td = worker_data_[q_idx];+ Queue& q = td.queue;+ unsigned w_idx;+ t = q.PushBackWithTag(call_worker_fn, pt.tag, w_idx);+ if (!t) {+ ps.tasks.push_back({q_idx, w_idx});+ td.EnsureAwake();+ }+ }+ }+}++// Run a single parallel loop in an existing parallel section. This+// maps directly onto SummonWorkers to create sufficient worker+// threads for the desired degree of parallelism, followed by+// dispatching the loop to those workers.++void RunInParallelSection(ThreadPoolParallelSection &ps,+ std::function<void(unsigned idx)> fn,+ unsigned n) override {+ PerThread* pt = GetPerThread();+ assert(pt->leading_par_section && "RunInParallel, but not in parallel section");+ assert((n > 1) && "Trivial parallel section; should be avoided by caller");++ // Publish the work to any existing workers in the parallel+ // section, and ensure it is visible to any new threads created+ // below.+ assert((!ps.current_loop) && "RunInParallelSection, but loop already active");+ ThreadPoolLoop loop{std::move(fn), n};+ ps.current_loop = &loop;++ // Increase the worker count if needed. Each worker will pick up+ // loops to execute from the current parallel section.+ const auto worker_fn = [&ps](unsigned my_idx) {+ while (ps.active) {+ if (!ps.current_loop) {+ onnxruntime::concurrency::SpinPause();+ } else {+ ps.workers_in_loop++;+ ThreadPoolLoop *work_item = ps.current_loop;+ if (work_item && my_idx < work_item->threads_needed) {+ work_item->fn(my_idx);+ }+ ps.workers_in_loop--;+ }+ }+ };+ SummonWorkers(*pt, ps, n, worker_fn);++ // Run work in the main thread+ loop.fn(0);++ // Wait for workers to exit the loop+ ps.current_loop = 0;+ while (ps.workers_in_loop) {+ onnxruntime::concurrency::SpinPause();+ }+}++// Run a single parallel loop _without_ a parallel section. This is a+// special case of RunInParallelSection, avoiding code paths for+// handing off multiple loops to the pool of workers.++void RunInParallel(std::function<void(unsigned idx)> fn, unsigned n) override {+ PerThread *pt = GetPerThread();+ ThreadPoolParallelSection ps;+ StartParallelSectionInternal(*pt, ps);++ // Summon workers to run the function (n is the desired maximum+ // degree of parallelism, including the main thread). Unlike the+ // multi-loop RunInParallelSection, this single-loop worker can run+ // fn directly without needing to receive it via ps.current_loop.+ SummonWorkers(*pt, ps, n, fn);++ // Run work in the main thread+ fn(0);++ // Wait for workers to exit the parallel section and hence to have+ // completed the loop (i.e., ps.tasks_finished matches the number of+ // tasks that have been created less the number successfully+ // revoked).+ EndParallelSectionInternal(*pt, ps);+}++void Cancel() override {+ cancelled_ = true;+ // If done_ is true, which means this object is being destructing.+ // Therefore worker_data_[i].thread could be NULL.+ if (!done_) {+ done_ = true;+ // Let each thread know it's been cancelled.+ for (size_t i = 0; i < worker_data_.size(); i++) {+ assert(worker_data_[i].thread != nullptr);+ worker_data_[i].thread->OnCancel();+ }+ }++ // Wake up the threads without work to let them exit on their own.+ WakeAllWorkersForExit();+}++int NumThreads() const EIGEN_FINAL {+ return num_threads_;+}++int CurrentThreadId() const EIGEN_FINAL {+ const PerThread* pt = const_cast<ThreadPoolTempl*>(this)->GetPerThread();+ if (pt->pool == this) {+ return pt->thread_id;+ }+ return -1;+}++ private:++#ifdef NDEBUG+ void AssertBounds(int, int) {+ }+#else+ void AssertBounds(int start, int end) {+ assert(start >= 0);+ assert(start < end); // non-zero sized partition+ assert(end <= num_threads_);+ }+#endif++ void ComputeCoprimes(int N, Eigen::MaxSizeVector<unsigned>* coprimes) {+ for (int i = 1; i <= N; i++) {+ unsigned a = i;+ unsigned b = N;+ // If GCD(a, b) == 1, then a and b are coprimes.+ while (b != 0) {+ unsigned tmp = a;+ a = b;+ b = tmp % b;+ }+ if (a == 1) {+ coprimes->push_back(i);+ }+ }+ }++ typedef typename Environment::EnvThread Thread;+ struct WorkerData;++ // PerThread objects are allocated in thread-local storage and allocated+ // on the thread's first call to GetPerThread. The object should+ // remain trivially-destructable, with other state placed in the+ // WorkerData objects that are allocated and cleaned-up explicitly.+ //+ // PerThread objects are allocated for all threads that submit work to+ // the thread pool, in addition to threads within the pool.+ //+ // In contrast, the WorkerData objects are allocated only for the+ // threads in the pool, and their lifetime is managed along with the+ // pool.++ struct PerThread {+ constexpr PerThread() : pool(nullptr) {+ }+ ThreadPoolTempl* pool; // Parent pool, or null for normal threads.+ uint64_t rand{0}; // Random generator state.+ int thread_id{-1}; // Worker thread index in pool.+ Tag tag{}; // Work item tag used to identify this thread.+ bool leading_par_section{false}; // Leading a parallel section (used only for asserts)+ };++ static_assert(std::is_trivially_destructible<PerThread>::value,+ "Per-thread state should be trivially destructible");++ struct WorkerData {+ constexpr WorkerData() : thread(), queue() {+ }+ std::unique_ptr<Thread> thread;+ Queue queue;++ // Each thread has a status, available read-only without locking, and protected+ // by the mutex field below for updates. The status is used for three+ // purposes:+ //+ // 1. To identify threads that are good candidates to push work to.+ // We prefer to push work to threads that are actively spinning (no need+ // for an OS wake-up, and no need for current work to finish). After that, we+ // prefer to push work to threads that are blocked (no need to wait for the+ // current work to finish).+ //+ // 2. To identify threads that are good candidates to steal work from. We+ // prefer to steal work from threads that are active outside the worker loop.+ // This avoids "snatching" new work away from a thread that has just been+ // given it but not yet noticed.+ //+ // 3. When pushing work to a thread, we use the status read-only to identify+ // when we need to wake the thread. This read-only check avoids the+ // need for mutex / condvar operations in the case where the thread pool+ // remains busy.++ enum class ThreadStatus : uint8_t {+ Spinning, // Spinning in the work loop, and other cases (initialization) where+ // the thread will soon be in the loop+ Active, // Running user code, not waiting for work+ Blocking, // In the process of blocking; may no longer notice work pushed to it+ Blocked, // Blocked on cv+ Waking, // Not yet back in the worker loop, but wake-up notification sent+ };++ ThreadStatus GetStatus() const {+ return status;+ }++ // State transitions, called from other threads++ void EnsureAwake() {+ ThreadStatus seen = status;+ if (seen == ThreadStatus::Blocking ||+ seen == ThreadStatus::Blocked) {+ std::unique_lock<OrtMutex> lk(mutex);+ // Blocking state exists only transiently during the SetBlock() method+ // while holding the lock. We may observe it at the start of this+ // function, but after acquiring the lock then the target thread+ // will either be blocked or not.+ seen = status;+ assert(seen != ThreadStatus::Blocking);+ if (seen == ThreadStatus::Blocked) {+ status = ThreadStatus::Waking;+ cv.notify_one();+ }+ }+ }++ // State transitions, called only from the thread itself++ void SetActive() {+ std::unique_lock<OrtMutex> lk(mutex);+ status = ThreadStatus::Active;+ }++ void SetSpinning() {+ std::unique_lock<OrtMutex> lk(mutex);+ status = ThreadStatus::Spinning;+ }++ void SetBlocked(std::function<bool()> should_block,+ std::function<void()> post_block) {+ std::unique_lock<OrtMutex> lk(mutex);+ assert(status == ThreadStatus::Spinning);+ status = ThreadStatus::Blocking;+ if (should_block()) {+ status = ThreadStatus::Blocked;+ while (status == ThreadStatus::Blocked) {+ cv.wait(lk);+ }+ post_block();+ }+ status = ThreadStatus::Spinning;+ }++ private:+ std::atomic<ThreadStatus> status{ThreadStatus::Spinning};+ OrtMutex mutex;+ OrtCondVar cv;+ };++ Environment& env_;+ const int num_threads_;+ const bool allow_spinning_;+ const bool set_denormal_as_zero_;+ Eigen::MaxSizeVector<WorkerData> worker_data_;+ Eigen::MaxSizeVector<Eigen::MaxSizeVector<unsigned>> all_coprimes_;+ std::atomic<unsigned> blocked_; // Count of blocked workers, used as a termination condition+ std::atomic<bool> done_;+ std::atomic<bool> cancelled_;++ // Allow control over how many bits to use in each entry in good_worker_hints_.+ // We reduce this below the full 64-bit word size for two reasons. First, it+ // helps test coverage on machines without 64 vCPUS. Second, it lets us+ // reduce contention by having different threads start work searching for hints+ // at different locations in the bitmap.++ static const unsigned bits_per_hint_word_ = 4;+ unsigned num_hint_words_;+ std::unique_ptr<std::atomic<uint64_t>[]> good_worker_hints_;++ // Wake any blocked workers so that they can cleanly exit WorkerLoop(). For an+ // abrupt exit, cancelled_==true and threads will exit their worker loops. For+ // a clean exit, each thread will observe (1) done_ set, indicating that the+ // destructor has been called, (2) all threads blocked, and (3) no+ // items in the work queues.++ void WakeAllWorkersForExit() {+ for (auto &td: worker_data_) {+ td.EnsureAwake();+ }+ }++ // Main worker thread loop.+ void WorkerLoop(int thread_id) {+ PerThread* pt = GetPerThread();+ WorkerData& td = worker_data_[thread_id];+ Queue& q = td.queue;+ bool should_exit = false;+ pt->pool = this;+ pt->rand = GlobalThreadIdHash();+ pt->thread_id = thread_id;++ assert(td.GetStatus() == WorkerData::ThreadStatus::Spinning);+ SetGoodWorkerHint(thread_id, true /* Is good */);++ const int log2_spin = 20;+ const int spin_count = allow_spinning_ ? (1ull<<log2_spin) : 0;+ const int steal_count = spin_count/100;++ SetDenormalAsZero(set_denormal_as_zero_);++ while (!cancelled_ && !should_exit) {+ Task t = q.PopFront();+ if (!t) {+ // Spin waiting for work. We indicate, via SetGOodWorkerHint that we are+ // spinning. This will bias other threads toward pushing work to our queue.+ // In addition, priodically make a best-effort attempt to steal from other+ // threads which are not themselves spinning.++ SetGoodWorkerHint(thread_id, true);+ for (int i = 0; i < spin_count && !t && !cancelled_ && !done_; i++) {+ t = ((i+1)%steal_count == 0) ? TrySteal() : q.PopFront();+ onnxruntime::concurrency::SpinPause();+ }+ SetGoodWorkerHint(thread_id, false);++ if (!t) {+ // No work passed to us while spinning; make a further full attempt to+ // steal work from other threads prior to blocking.+ if (num_threads_ != 1) {+ t = Steal(true /* true => check all queues */);+ }+ if (!t) {+ td.SetBlocked(+ // Pre-block test+ [&]() -> bool {+ bool should_block = true;+ // We already did a best-effort emptiness check when stealing; now+ // do a full check prior to blocking.+ int victim = NonEmptyQueueIndex();+ if (victim != -1) {+ should_block = false;+ if (!cancelled_) {+ t = worker_data_[victim].queue.PopBack();+ }+ }+ // Number of blocked threads is used as termination condition.+ // If we are shutting down and all worker threads blocked without work,+ // that's we are done.+ if (should_block) {+ blocked_++;+ if (done_ && blocked_ == static_cast<unsigned>(num_threads_)) {+ should_block = false;+ // Almost done, but need to re-check queues.+ // Consider that all queues are empty and all worker threads are preempted+ // right after incrementing blocked_ above. Now a free-standing thread+ // submits work and calls destructor (which sets done_). If we don't+ // re-check queues, we will exit leaving the work unexecuted.+ if (NonEmptyQueueIndex() != -1) {+ // Note: we must not pop from queues before we decrement blocked_,+ // otherwise the following scenario is possible. Consider that instead+ // of checking for emptiness we popped the only element from queues.+ // Now other worker threads can start exiting, which is bad if the+ // work item submits other work. So we just check emptiness here,+ // which ensures that all worker threads exit at the same time.+ blocked_--;+ } else {+ should_exit = true;+ }+ }+ }+ return should_block;+ },+ // Post-block update (executed only if we blocked)+ [&]() {+ blocked_--;+ });+ }+ }+ }+ if (t) {+ td.SetActive();+ t();+ td.SetSpinning();+ }+ }++ // Whichever thread(s) observe the termination conditions are responsible for waking+ // any other threads that have remained blocked.+ if (should_exit) {+ WakeAllWorkersForExit();+ }+ }++ // Steal tries to steal work from other worker threads in the range [start,+ // limit) in best-effort manner. We make two passes over the threads:+ //+ // - round 0 : we attempt to steal from threads that are running in+ // user code (ThreadStatus::Active). The intuition behind this is that+ // the thread is busy with other work, and that by preferring to+ // steel from busy victims we will avoid "snatching" work from a+ // thread which is just about to notice the work itself.+ //+ // - round 1 : we steal work from any thread, including those which claim+ // to be spinning. In these cases, even though the victim thread is+ // looking for work itself, it may have been pre-empted.++ Task Steal(bool check_all) {+ PerThread* pt = GetPerThread();+ unsigned size = static_cast<unsigned>(num_threads_);+ unsigned r = Rand(&pt->rand);+ unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()];++ for (int round = 0; round < 2; round++) {+ unsigned victim = r % size;+ for (unsigned i = 0; i < size; i++) {+ assert(victim < size);+ if (round == 1 ||+ worker_data_[victim].GetStatus() == WorkerData::ThreadStatus::Active) {+ Task t = worker_data_[victim].queue.PopBack();+ if (t) {+ return t;+ }+ }+ if (!check_all) {+ return Task();+ }+ victim += inc;+ if (victim >= size) {+ victim -= size;+ }+ }+ }++ return Task();+ }++ Task TrySteal() {+ return Steal(false);+ }++ int NonEmptyQueueIndex() {+ PerThread* pt = GetPerThread();+ const unsigned size = static_cast<unsigned>(worker_data_.size());+ unsigned r = Rand(&pt->rand);+ unsigned inc = all_coprimes_[size - 1][r % all_coprimes_[size - 1].size()];+ unsigned victim = r % size;+ for (unsigned i = 0; i < size; i++) {+ if (!worker_data_[victim].queue.Empty()) {+ return victim;+ }+ victim += inc;+ if (victim >= size) {+ victim -= size;+ }+ }+ return -1;+ }++ static EIGEN_STRONG_INLINE uint64_t GlobalThreadIdHash() {+ return std::hash<std::thread::id>()(std::this_thread::get_id());+ }++ static EIGEN_STRONG_INLINE PerThread* GetPerThread() {+ static thread_local PerThread per_thread_;+ PerThread* pt = &per_thread_;+ return pt;+ }++ static EIGEN_STRONG_INLINE unsigned Rand(uint64_t* state) {+ uint64_t current = *state;+ // Update the internal state+ *state = current * 6364136223846793005ULL + 0xda3e39cb94b95bdbULL;+ // Generate the random output (using the PCG-XSH-RS scheme)+ return static_cast<unsigned>((current ^ (current >> 22)) >> (22 + (current >> 61)));+ }+};++ } // namespace concurrency++} // namespace onnxruntimediff --git a/include/onnxruntime/core/platform/ort_mutex.h b/include/onnxruntime/core/platform/ort_mutex.hnew file mode 100644index 0000000..e24665f--- /dev/null+++ b/include/onnxruntime/core/platform/ort_mutex.h@@ -0,0 +1,189 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#ifdef _WIN32+#include <Windows.h>+#include <mutex>+namespace onnxruntime {+// Q: Why OrtMutex is better than std::mutex+// A: OrtMutex supports static initialization but std::mutex doesn't. Static initialization helps us prevent the "static+// initialization order problem".++// Q: Why std::mutex can't make it?+// A: VC runtime has to support Windows XP at ABI level. But we don't have such requirement.++// Q: Is OrtMutex faster than std::mutex?+// A: Sure++class OrtMutex {+ private:+ SRWLOCK data_ = SRWLOCK_INIT;++ public:+ constexpr OrtMutex() = default;+ // SRW locks do not need to be explicitly destroyed.+ ~OrtMutex() = default;+ OrtMutex(const OrtMutex&) = delete;+ OrtMutex& operator=(const OrtMutex&) = delete;+ void lock() { AcquireSRWLockExclusive(native_handle()); }+ bool try_lock() noexcept { return TryAcquireSRWLockExclusive(native_handle()) == TRUE; }+ void unlock() noexcept { ReleaseSRWLockExclusive(native_handle()); }+ using native_handle_type = SRWLOCK*;++ __forceinline native_handle_type native_handle() { return &data_; }+};++class OrtCondVar {+ CONDITION_VARIABLE native_cv_object = CONDITION_VARIABLE_INIT;++ public:+ constexpr OrtCondVar() noexcept = default;+ ~OrtCondVar() = default;++ OrtCondVar(const OrtCondVar&) = delete;+ OrtCondVar& operator=(const OrtCondVar&) = delete;++ void notify_one() noexcept { WakeConditionVariable(&native_cv_object); }+ void notify_all() noexcept { WakeAllConditionVariable(&native_cv_object); }++ void wait(std::unique_lock<OrtMutex>& lk) {+ if (SleepConditionVariableSRW(&native_cv_object, lk.mutex()->native_handle(), INFINITE, 0) != TRUE) {+ std::terminate();+ }+ }+ template <class _Predicate>+ void wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred);++ /**+ * returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns+ * cv_status::no_timeout.+ * @param cond_mutex A unique_lock<OrtMutex> object.+ * @param rel_time A chrono::duration object that specifies the amount of time before the thread wakes up.+ * @return returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns+ * cv_status::no_timeout+ */+ template <class Rep, class Period>+ std::cv_status wait_for(std::unique_lock<OrtMutex>& cond_mutex, const std::chrono::duration<Rep, Period>& rel_time);+ using native_handle_type = CONDITION_VARIABLE*;++ native_handle_type native_handle() { return &native_cv_object; }++ private:+ void timed_wait_impl(std::unique_lock<OrtMutex>& __lk,+ std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>);+};++template <class _Predicate>+void OrtCondVar::wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred) {+ while (!__pred()) wait(__lk);+}++template <class Rep, class Period>+std::cv_status OrtCondVar::wait_for(std::unique_lock<OrtMutex>& cond_mutex,+ const std::chrono::duration<Rep, Period>& rel_time) {+ // TODO: is it possible to use nsync_from_time_point_ ?+ using namespace std::chrono;+ if (rel_time <= duration<Rep, Period>::zero())+ return std::cv_status::timeout;+ using SystemTimePointFloat = time_point<system_clock, duration<long double, std::nano> >;+ using SystemTimePoint = time_point<system_clock, nanoseconds>;+ SystemTimePointFloat max_time = SystemTimePoint::max();+ steady_clock::time_point steady_now = steady_clock::now();+ system_clock::time_point system_now = system_clock::now();+ if (max_time - rel_time > system_now) {+ nanoseconds remain = duration_cast<nanoseconds>(rel_time);+ if (remain < rel_time)+ ++remain;+ timed_wait_impl(cond_mutex, system_now + remain);+ } else+ timed_wait_impl(cond_mutex, SystemTimePoint::max());+ return steady_clock::now() - steady_now < rel_time ? std::cv_status::no_timeout : std::cv_status::timeout;+}+} // namespace onnxruntime+#else+#include "nsync.h"+#include <mutex> //for unique_lock+#include <condition_variable> //for cv_status+namespace onnxruntime {++class OrtMutex {+ nsync::nsync_mu data_ = NSYNC_MU_INIT;++ public:+ constexpr OrtMutex() = default;+ ~OrtMutex() = default;+ OrtMutex(const OrtMutex&) = delete;+ OrtMutex& operator=(const OrtMutex&) = delete;++ void lock() { nsync::nsync_mu_lock(&data_); }+ bool try_lock() noexcept { return nsync::nsync_mu_trylock(&data_) == 0; }+ void unlock() noexcept { nsync::nsync_mu_unlock(&data_); }++ using native_handle_type = nsync::nsync_mu*;+ native_handle_type native_handle() { return &data_; }+};++class OrtCondVar {+ nsync::nsync_cv native_cv_object = NSYNC_CV_INIT;++ public:+ constexpr OrtCondVar() noexcept = default;++ ~OrtCondVar() = default;+ OrtCondVar(const OrtCondVar&) = delete;+ OrtCondVar& operator=(const OrtCondVar&) = delete;++ void notify_one() noexcept { nsync::nsync_cv_signal(&native_cv_object); }+ void notify_all() noexcept { nsync::nsync_cv_broadcast(&native_cv_object); }++ void wait(std::unique_lock<OrtMutex>& lk);+ template <class _Predicate>+ void wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred);++ /**+ * returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns+ * cv_status::no_timeout.+ * @param cond_mutex A unique_lock<OrtMutex> object.+ * @param rel_time A chrono::duration object that specifies the amount of time before the thread wakes up.+ * @return returns cv_status::timeout if the wait terminates when Rel_time has elapsed. Otherwise, the method returns+ * cv_status::no_timeout+ */+ template <class Rep, class Period>+ std::cv_status wait_for(std::unique_lock<OrtMutex>& cond_mutex, const std::chrono::duration<Rep, Period>& rel_time);+ using native_handle_type = nsync::nsync_cv*;+ native_handle_type native_handle() { return &native_cv_object; }++ private:+ void timed_wait_impl(std::unique_lock<OrtMutex>& __lk,+ std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds>);+};++template <class _Predicate>+void OrtCondVar::wait(std::unique_lock<OrtMutex>& __lk, _Predicate __pred) {+ while (!__pred()) wait(__lk);+}++template <class Rep, class Period>+std::cv_status OrtCondVar::wait_for(std::unique_lock<OrtMutex>& cond_mutex,+ const std::chrono::duration<Rep, Period>& rel_time) {+ // TODO: is it possible to use nsync_from_time_point_ ?+ using namespace std::chrono;+ if (rel_time <= duration<Rep, Period>::zero())+ return std::cv_status::timeout;+ using SystemTimePointFloat = time_point<system_clock, duration<long double, std::nano> >;+ using SystemTimePoint = time_point<system_clock, nanoseconds>;+ SystemTimePointFloat max_time = SystemTimePoint::max();+ steady_clock::time_point steady_now = steady_clock::now();+ system_clock::time_point system_now = system_clock::now();+ if (max_time - rel_time > system_now) {+ nanoseconds remain = duration_cast<nanoseconds>(rel_time);+ if (remain < rel_time)+ ++remain;+ timed_wait_impl(cond_mutex, system_now + remain);+ } else+ timed_wait_impl(cond_mutex, SystemTimePoint::max());+ return steady_clock::now() - steady_now < rel_time ? std::cv_status::no_timeout : std::cv_status::timeout;+}+}; // namespace onnxruntime+#endifdiff --git a/include/onnxruntime/core/platform/threadpool.h b/include/onnxruntime/core/platform/threadpool.hnew file mode 100644index 0000000..126dd13--- /dev/null+++ b/include/onnxruntime/core/platform/threadpool.h@@ -0,0 +1,441 @@+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.++Licensed under the Apache License, Version 2.0 (the "License");+you may not use this file except in compliance with the License.+You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++Unless required by applicable law or agreed to in writing, software+distributed under the License is distributed on an "AS IS" BASIS,+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+See the License for the specific language governing permissions and+limitations under the License.+==============================================================================*/++/* Modifications Copyright (c) Microsoft. */++#pragma once+#include <string>+#include <vector>+#include <functional>+#include <memory>+#include "core/common/common.h"+#include "core/platform/env.h"++#include <functional>+#include <memory>++// ORT thread pool overview+// ------------------------+//+// The ORT thread pool implementation is split into two layers. This+// file provides the high-level component. See the accompanying+// comments in EigenNonBlockingThreadPool.h for the low-level+// component.+//+// threadpool.h defines the user-facing functions for use in+// operators. The main abstraction are parallel loops+// (ThreadPool::TryParallelFor*), although we also support scheduling+// of asynchronous tasks (ThreadPool::Schedule), and the construction+// of multi-loop parallel sections (ThreadPool::ParallelSection).+//+// This high level API is accessed via static methods on the+// ThreadPool class. These methods map the operations onto one of+// three low-level implementations: (#1) direct execution of the+// operations if there is no thread pool configured, (#2) execution of+// the operations using the modified Eigen threadpool, (#3) execution+// of the operations using OpenMP. Option #1 enables execution in+// simple settings without needing threads. Option #2 is the+// preferred approach for use in settings with parallelism.+//+// The high-level part of the thread pool is responsible for:+//+// - Exposing the desired degree of parallelism to user code, and to+// libraries such as MLAS. This lets the libraries tailor the+// extent to which they parallelize work.+//+// - Handling trivial cases (such as directly running parallel loops+// with only a single iteration, or with no iterations at all).+//+// - Deciding how to divide work efficiently between the threads+// available.+//+// The ThreadPool::TryParallelFor methods do this based on cost+// estimates supplied by the caller, and are designed to support+// loops with small amounts of work per iteration. The loop body is+// supplied as a function taking a [start,end) range of iterations+// to execute (avoiding the need for per-iteration std::function+// calls, or a reliance upon inlining to avoid those calls).+//+// ThreadPool::TrySimpleParallelFor uses a simpler single-iteration+// API based on the assumption that the caller has divided work to+// an appropriate granularity.+//+// - When used with the Eigen-based thread pool, the implementation of+// all of the loops maps down onto+// ThreadPool::ParallelForFixedBlockSizeScheduling. This method+// takes the degree of parallelism (d_of_p) and work distribution+// block size (from the cost-based heuristics), and creates a set of+// tasks in the underlying thread pool (via+// ThreadPool::RunInParallel).+//+// These tasks then run a loop which picks off batches of iterations+// from the user's code. The distribution of these batches is+// handled dynmamically via LoopCounter::ClaimIterations. This+// dynamic balancing behavior helps make performance robust to any+// variability in the execution time across iterations, and to+// situations such as multiple loops running concurrently on the+// same thread pool.+//+// - When running a series of loops inside a parallel section, the+// LoopCounter also helps obtain affinity between these loops (i.e.,+// iteration X of one loop will tend to run on the same thread that+// ran iteration X of prior loops). This locality helps improve hit+// rates in per-core caches across the series of short loops used in+// operators like GRU.+//+// There are some known areas for exploration here:+//+// - The cost-based heuristics were developed prior to recent changes+// to the thread pool. The heuristics seem to work well, but we+// should revisit the tuning periodically.+//+// - Can we unify the APIs for the different kinds of parallel loop?+//+// In particular, we may be able to replace the current use of+// TryBatchParallelFor with appropriate costs for each call site,+// and then use TryParallelFor. This would allow for more dynamic+// re-balancing of work between threads than the current+// ThreadPool::PartitionWork function provides.+//+// - Given the extensive modifications to original Eigen code, should+// we separate that out as a new class and remove the dependence on+// other Eigen components.++// This file use PIMPL to avoid having eigen headers here+namespace Eigen {+class Allocator;+class ThreadPoolInterface;+} // namespace Eigen++namespace onnxruntime {++struct TensorOpCost {+ double bytes_loaded;+ double bytes_stored;+ double compute_cycles;+};+++namespace concurrency {++template <typename Environment>+class ThreadPoolTempl;++class ExtendedThreadPoolInterface;+class LoopCounter;+class ThreadPoolParallelSection;++class ThreadPool {+ public:+#ifdef _WIN32+ using NAME_CHAR_TYPE = wchar_t;+#else+ using NAME_CHAR_TYPE = char;+#endif+ // Constructs a pool for running with with "degree_of_parallelism" threads with+ // specified "name". env->StartThread() is used to create individual threads+ // with the given ThreadOptions. If "low_latency_hint" is true the thread pool+ // implementation may use it as a hint that lower latency is preferred at the+ // cost of higher CPU usage, e.g. by letting one or more idle threads spin+ // wait. Conversely, if the threadpool is used to schedule high-latency+ // operations like I/O the hint should be set to false.+ //+ // REQUIRES: degree_of_parallelism > 0+ ThreadPool(Env* env,+ const ThreadOptions& thread_options,+ const NAME_CHAR_TYPE* name,+ int degree_of_parallelism,+ bool low_latency_hint);++ // Waits until all scheduled work has finished and then destroy the+ // set of threads.+ ~ThreadPool();++ // Start and end a multi-loop parallel section. Parallel loops can+ // be executed directly (without using this API), but entering a+ // parallel section allows the runtime system to amortize loop+ // entry/exit costs over multiple loops, and allows it to promote+ // affinity between corresponding iterations of different loops.+ //+ // Multi-loop sections would typically be used in cases where a+ // series of loops executes without much code in between them, and+ // where it is impractical to refactor code into a single loop. For+ // instance:+ //+ // {+ // onnxruntime::concurrency::ThreadPoool::ParallelSection ps(tp);+ // for (int x = 0; x < seq_len; x++) {+ // TrySimpleParallelFor(tp, 16, [&]() { ... });+ // }+ // }+ //+ // The parallel section is entered via the constructor of+ // ThreadPool::ParallelSection, and exited via the destructor.+ // Currently, thread-local state is used to track whether or not the+ // current thread is inside a parallel section. In contrast to+ // handling parallel section objects explicitly in user code, this+ // approach allows code such as MLAS to operate with/without the use+ // of parallel sections.+ //+ // Parallel sections are only implemented with the Eigen threadpool.+ // They have no effect when using OpenMP.+ //+ // Parallel sections may not be nested, and may not be used inside+ // parallel loops.++ class ParallelSection {+ public:+ explicit ParallelSection(ThreadPool *tp);+ ~ParallelSection();++ private:+ friend class ThreadPool;++ // Owning reference for the underlying ThreadPoolParallelSection+ // which implements the thread management. We use an explicit+ // deleter here so that the definition of+ // ThreadPoolParallelSection does not need to be available at this+ // point to avoid a dependence on the Eigen headers.+ std::unique_ptr<ThreadPoolParallelSection, void(*)(ThreadPoolParallelSection*)>+ ps_{nullptr, [](ThreadPoolParallelSection*){}};+#ifndef _OPENMP+ ThreadPool *tp_;+#endif+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ParallelSection);++ // Non-owning reference to the current thread's paralel section+ // (or nullptr outside parallel sections).+ static thread_local ParallelSection *current_parallel_section;+ static_assert(std::is_trivially_destructible<decltype(current_parallel_section)>::value,+ "Per-thread state should be trivially destructible");+ };++ // Schedules fn() for execution in the pool of threads. The function may run+ // synchronously if it cannot be enqueued. This will occur if the thread pool's+ // degree-of-parallelism is 1, but it may also occur for implementation-dependent+ // reasons such as if queues used for buffering work are full.+ static void Schedule(ThreadPool* tp,+ std::function<void()> fn) {+ if (tp) {+ tp->Schedule(fn);+ } else {+ fn();+ }+ }++ // ParallelFor shards the "total" units of work assuming each unit of work+ // having roughly "cost_per_unit" cost, in cycles. Each unit of work is+ // indexed 0, 1, ..., total - 1. Each shard contains 1 or more units of work+ // and the total cost of each shard is roughly the same.+ //+ // "cost_per_unit" is an estimate of the number of CPU cycles (or nanoseconds+ // if not CPU-bound) to complete a unit of work. Overestimating creates too+ // many shards and CPU time will be dominated by per-shard overhead, such as+ // Context creation. Underestimating may not fully make use of the specified+ // parallelism, and may also cause inefficiencies due to load balancing+ // issues and stragglers.++ static void TryParallelFor(ThreadPool* tp, std::ptrdiff_t total, double cost_per_unit,+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn) {+ TryParallelFor(tp, total, TensorOpCost{0, 0, static_cast<double>(cost_per_unit)}, fn);+ }++ static void TryParallelFor(ThreadPool* tp, std::ptrdiff_t total, const TensorOpCost& cost_per_unit,+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn);++ // Directly schedule the 'total' tasks to the underlying threadpool, without+ // cutting them by halves++ inline static void TrySimpleParallelFor(ThreadPool* tp, std::ptrdiff_t total,+ const std::function<void(std::ptrdiff_t)>& fn) {+#ifdef _OPENMP+ ORT_UNUSED_PARAMETER(tp);+#pragma omp parallel for+ for (std::ptrdiff_t i = 0; i < total; ++i) {+ fn(i);+ }+#else+ if (tp != nullptr) {+ tp->SimpleParallelFor(total, fn);+ } else {+ for (std::ptrdiff_t i = 0; i < total; ++i) {+ // In many cases, fn can be inlined here.+ fn(i);+ }+ }+#endif+ }++ /**+ * Tries to call the given function in parallel, with calls split into (num_batches) batches.+ *\param num_batches If it is zero, it will be replaced to the value of DegreeOfParallelism().+ *\param fn A std::function or STL style functor with signature of "void f(int32_t);"+ * Pitfall: Caller should cap `num_batches` to a reasonable value based on the cost of `fn` and the value of `total`.+ *For example, if fn is as simple as: int sum=0; fn = [&](int i){sum +=i;} and `total` is 100, then num_batches should+ *be just 1.+ *+ * ```+ **/+ template <typename F>+ inline static void TryBatchParallelFor(ThreadPool* tp, std::ptrdiff_t total, F&& fn, std::ptrdiff_t num_batches) {+#ifdef _OPENMP+ ORT_UNUSED_PARAMETER(tp);+ ORT_UNUSED_PARAMETER(num_batches);+#pragma omp parallel for+ for (std::ptrdiff_t i = 0; i < total; ++i) {+ fn(i);+ }+#else+ if (tp == nullptr) {+ for (std::ptrdiff_t i = 0; i < total; ++i) {+ // In many cases, fn can be inlined here.+ fn(i);+ }+ return;+ }+ if (total <= 0)+ return;++ if (total == 1) {+ fn(0);+ return;+ }++ if (num_batches <= 0) {+ num_batches = std::min<ptrdiff_t>(total, DegreeOfParallelism(tp));+ }++ if (num_batches <= 1) {+ for (int i = 0; i < total; i++) {+ fn(i);+ }+ return;+ }++ tp->SimpleParallelFor(num_batches, [&](std::ptrdiff_t batch_index) {+ auto work = PartitionWork(batch_index, num_batches, total);+ for (std::ptrdiff_t i = work.start; i < work.end; i++) {+ fn(i);+ }+ });+#endif+ }++ struct WorkInfo {+ std::ptrdiff_t start;+ std::ptrdiff_t end;+ };++ /** Calculate the start and end offsets for a batch.+ @remarks Based on MlasPartitionWork+ */+ static WorkInfo PartitionWork(std::ptrdiff_t batch_idx, std::ptrdiff_t num_batches, std::ptrdiff_t total_work) {+ const std::ptrdiff_t work_per_batch = total_work / num_batches;+ const std::ptrdiff_t work_per_batch_extra = total_work % num_batches;++ WorkInfo info;+ if (batch_idx < work_per_batch_extra) {+ info.start = (work_per_batch + 1) * batch_idx;+ info.end = info.start + work_per_batch + 1;+ } else {+ info.start = work_per_batch * batch_idx + work_per_batch_extra;+ info.end = info.start + work_per_batch;+ }++ return info;+ }++ //......................................................................+ //+ // The following static methods take into account whether OpenMP is+ // enabled/disabled, and if the thread pool pointer is nullptr+ // during sequential execution.++ // Provide a hint to the caller for whether or not to parallelize+ // work. This lets a caller switch to a sequential version of an+ // algorithm rather than using calls via the ParallelFor functions.++ static bool ShouldParallelize(const ThreadPool* tp);++ // Return the degree of parallelism that code should assume when using the thread pool.+ // It decouples the degree of parallelism for use with the thread pool from+ // the implementation choice of whether this matches the number of threads created in+ // the pool.+ //+ // Currently, a loop with degree-of-parallelism N is supported by a pool of N-1 threads+ // working in combination with the thread initiating the loop.+ static int DegreeOfParallelism(const ThreadPool* tp);++ ORT_DISALLOW_COPY_AND_ASSIGNMENT(ThreadPool);++ private:+ friend class LoopCounter;++ // Returns the number of threads created in the pool. This may be different from the+ // value returned by DegreeOfParallelism to code using the pool.+ int NumThreads() const;++ // Returns current thread id between 0 and NumThreads() - 1, if called from a+ // thread in the pool. Returns -1 otherwise.+ int CurrentThreadId() const;++ // Run fn with up to n degree-of-parallelism enlisting the thread pool for+ // help. The degree-of-parallelism includes the caller, and so if n==1+ // then the function will run directly in the caller. The fork-join+ // synchronization is handled in the thread pool, and so any state captured+ // by fn() is safe from concurrent access once RunWithHelp returns.+ void RunInParallel(std::function<void(unsigned idx)> fn, unsigned n);++ // Divides the work represented by the range [0, total) into k shards.+ // Calls fn(i*block_size, (i+1)*block_size) from the ith shard (0 <= i < k).+ // Each shard may be executed on a different thread in parallel, depending on+ // the number of threads available in the pool.+ // When (i+1)*block_size > total, fn(i*block_size, total) is called instead.+ // Requires 0 < block_size <= total.+ void ParallelForFixedBlockSizeScheduling(std::ptrdiff_t total, std::ptrdiff_t block_size,+ const std::function<void(std::ptrdiff_t, std::ptrdiff_t)>& fn);++ // Return whether or not the calling thread should run a loop of+ // num_iterations divided in chunks of block_size in parallel. If not,+ // the caller should run the loop sequentially.+ bool ShouldParallelizeLoop(const std::ptrdiff_t num_iterations,+ const std::ptrdiff_t block_size = 1) const;++ // Internal (non-static) parallel loop methods. Unlike the public static methods,+ // these will not handle the cases of OpenMP builds. or builds without a threadpool.+ void ParallelFor(std::ptrdiff_t total, double cost_per_unit,+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t last)>& fn);++ void ParallelFor(std::ptrdiff_t total, const TensorOpCost& cost_per_unit,+ const std::function<void(std::ptrdiff_t first, std::ptrdiff_t)>& fn);++ void SimpleParallelFor(std::ptrdiff_t total, const std::function<void(std::ptrdiff_t)>& fn);++ void Schedule(std::function<void()> fn);++ ThreadOptions thread_options_;++ // If a thread pool is created with degree_of_parallelism != 1 then an underlying+ // EigenThreadPool is used to create OS threads and handle work distribution to them.+ // If degree_of_parallelism == 1 then underlying_threadpool_ is left as nullptr+ // and parallel work is run directly by the caller.+ ExtendedThreadPoolInterface* underlying_threadpool_ = nullptr;++ // If used, underlying_threadpool_ is instantiated and owned by the ThreadPool.+ std::unique_ptr<ThreadPoolTempl<Env> > extended_eigen_threadpool_;+};++} // namespace concurrency+} // namespace onnxruntimediff --git a/include/onnxruntime/core/platform/tracing.h b/include/onnxruntime/core/platform/tracing.hnew file mode 100644index 0000000..fb61632--- /dev/null+++ b/include/onnxruntime/core/platform/tracing.h@@ -0,0 +1,9 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <windows.h>+#include <TraceLoggingProvider.h>++TRACELOGGING_DECLARE_PROVIDER(telemetry_provider_handle);diff --git a/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.hnew file mode 100644index 0000000..2ee92a8--- /dev/null+++ b/include/onnxruntime/core/platform/windows/TraceLoggingConfig.h@@ -0,0 +1,81 @@+// Copyright (c) Microsoft Corporation.+// Licensed under the MIT License.++/* +++Module Name:+ TraceLoggingConfig.h+Abstract:+ Macro definitions used by this project's TraceLogging ETW providers:+ - Configuration macros that select the ETW Provider Groups to be used by+ this project.+ - Constants for tags that are commonly used in Microsoft's+ TraceLogging-based ETW.+ Different versions of this file use different definitions for the+ TraceLoggingOption configuration macros. The definitions in this file are+ empty. As a result, providers using this configuration file will not join+ any ETW Provider Groups and will not be given any special treatment by+ group-sensitive ETW listeners.+Environment:+ User mode or kernel mode.+--*/++#pragma once++// Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition+// in this file configures the provider as a normal (non-telemetry) provider.+#ifndef TraceLoggingOptionMicrosoftTelemetry+#define TraceLoggingOptionMicrosoftTelemetry() \+ TraceLoggingOptionGroup(0000000000, 00000, 00000, 0000, 0000, 0000, 0000, 0000, 000, 0000, 0000)+ // Empty definition for TraceLoggingOptionMicrosoftTelemetry+#endif++// Configuration macro for use in TRACELOGGING_DEFINE_PROVIDER. The definition+// in this file configures the provider as a normal (non-telemetry) provider.+#define TraceLoggingOptionWindowsCoreTelemetry() \+ // Empty definition for TraceLoggingOptionWindowsCoreTelemetry++// Event privacy tags. Use the PDT macro values for the tag parameter, e.g.:+// TraceLoggingWrite(...,+// TelemetryPrivacyDataTag(PDT_BrowsingHistory | PDT_ProductAndServiceUsage),+// ...);+#define TelemetryPrivacyDataTag(tag) TraceLoggingUInt64((tag), "PartA_PrivTags")+#define PDT_BrowsingHistory 0x0000000000000002u+#define PDT_DeviceConnectivityAndConfiguration 0x0000000000000800u+#define PDT_InkingTypingAndSpeechUtterance 0x0000000000020000u+#define PDT_ProductAndServicePerformance 0x0000000001000000u+#define PDT_ProductAndServiceUsage 0x0000000002000000u+#define PDT_SoftwareSetupAndInventory 0x0000000080000000u++// Event categories specified via keywords, e.g.:+// TraceLoggingWrite(...,+// TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),+// ...);+#define MICROSOFT_KEYWORD_CRITICAL_DATA 0x0000800000000000 // Bit 47+#define MICROSOFT_KEYWORD_MEASURES 0x0000400000000000 // Bit 46+#define MICROSOFT_KEYWORD_TELEMETRY 0x0000200000000000 // Bit 45+#define MICROSOFT_KEYWORD_RESERVED_44 0x0000100000000000 // Bit 44 (reserved for future assignment)++// Event categories specified via event tags, e.g.:+// TraceLoggingWrite(...,+// TraceLoggingEventTag(MICROSOFT_EVENTTAG_REALTIME_LATENCY),+// ...);+#define MICROSOFT_EVENTTAG_DROP_USER_IDS 0x00008000+#define MICROSOFT_EVENTTAG_AGGREGATE 0x00010000+#define MICROSOFT_EVENTTAG_DROP_PII_EXCEPT_IP 0x00020000+#define MICROSOFT_EVENTTAG_COSTDEFERRED_LATENCY 0x00040000+#define MICROSOFT_EVENTTAG_CORE_DATA 0x00080000+#define MICROSOFT_EVENTTAG_INJECT_XTOKEN 0x00100000+#define MICROSOFT_EVENTTAG_REALTIME_LATENCY 0x00200000+#define MICROSOFT_EVENTTAG_NORMAL_LATENCY 0x00400000+#define MICROSOFT_EVENTTAG_CRITICAL_PERSISTENCE 0x00800000+#define MICROSOFT_EVENTTAG_NORMAL_PERSISTENCE 0x01000000+#define MICROSOFT_EVENTTAG_DROP_PII 0x02000000+#define MICROSOFT_EVENTTAG_HASH_PII 0x04000000+#define MICROSOFT_EVENTTAG_MARK_PII 0x08000000++// Field categories specified via field tags, e.g.:+// TraceLoggingWrite(...,+// TraceLoggingString(szUser, "UserName", "User's name", MICROSOFT_FIELDTAG_HASH_PII),+// ...);+#define MICROSOFT_FIELDTAG_DROP_PII 0x04000000+#define MICROSOFT_FIELDTAG_HASH_PII 0x08000000diff --git a/include/onnxruntime/core/platform/windows/readme.txt b/include/onnxruntime/core/platform/windows/readme.txtnew file mode 100644index 0000000..f1a436f--- /dev/null+++ b/include/onnxruntime/core/platform/windows/readme.txt@@ -0,0 +1,2 @@+copied from minkernel/published/internal/telemetry/open_source/TraceLoggingConfig.h+this is the official open source edition for these configuration settings\ No newline at end of filediff --git a/include/onnxruntime/core/providers/acl/acl_provider_factory.h b/include/onnxruntime/core/providers/acl/acl_provider_factory.hnew file mode 100644index 0000000..0dc0ec2--- /dev/null+++ b/include/onnxruntime/core/providers/acl/acl_provider_factory.h@@ -0,0 +1,18 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param use_arena zero: false. non-zero: true.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ACL, _In_ OrtSessionOptions* options, int use_arena)+ORT_ALL_ARGS_NONNULL;++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.hnew file mode 100644index 0000000..35276db--- /dev/null+++ b/include/onnxruntime/core/providers/armnn/armnn_provider_factory.h@@ -0,0 +1,18 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param use_arena zero: false. non-zero: true.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ArmNN, _In_ OrtSessionOptions* options, int use_arena)+ORT_ALL_ARGS_NONNULL;++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.hnew file mode 100644index 0000000..66f2589--- /dev/null+++ b/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h@@ -0,0 +1,18 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param use_arena zero: false. non-zero: true.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena)+ORT_ALL_ARGS_NONNULL;++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.hnew file mode 100644index 0000000..81b5477--- /dev/null+++ b/include/onnxruntime/core/providers/cuda/cuda_provider_factory.h@@ -0,0 +1,17 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param device_id cuda device id, starts from zero.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/dml/dml_provider_factory.h b/include/onnxruntime/core/providers/dml/dml_provider_factory.hnew file mode 100644index 0000000..071e7f0--- /dev/null+++ b/include/onnxruntime/core/providers/dml/dml_provider_factory.h@@ -0,0 +1,47 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma warning(push)+#pragma warning(disable : 4201) // nonstandard extension used: nameless struct/union+#include <d3d12.h>+#pragma warning(pop)++#ifdef __cplusplus+ #include <DirectML.h>+#else+ struct IDMLDevice;+ typedef struct IDMLDevice IDMLDevice;+#endif++// Windows pollutes the macro space, causing a build break in constants.h.+#undef OPTIONAL++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * Creates a DirectML Execution Provider which executes on the hardware adapter with the given device_id, also known as+ * the adapter index. The device ID corresponds to the enumeration order of hardware adapters as given by+ * IDXGIFactory::EnumAdapters. A device_id of 0 always corresponds to the default adapter, which is typically the+ * primary display GPU installed on the system. A negative device_id is invalid.+*/+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_DML, _In_ OrtSessionOptions* options, int device_id);++/**+ * Creates a DirectML Execution Provider using the given DirectML device, and which executes work on the supplied D3D12+ * command queue. The DirectML device and D3D12 command queue must have the same parent ID3D12Device, or an error will+ * be returned. The D3D12 command queue must be of type DIRECT or COMPUTE (see D3D12_COMMAND_LIST_TYPE). If this+ * function succeeds, the inference session maintains a strong reference on both the dml_device and the command_queue+ * objects.+ * See also: DMLCreateDevice+ * See also: ID3D12Device::CreateCommandQueue+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProviderEx_DML, _In_ OrtSessionOptions* options,+ _In_ IDMLDevice* dml_device, _In_ ID3D12CommandQueue* cmd_queue);++#ifdef __cplusplus+}+#endif\ No newline at end of filediff --git a/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.h b/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.hnew file mode 100644index 0000000..b87780c--- /dev/null+++ b/include/onnxruntime/core/providers/dnnl/dnnl_provider_factory.h@@ -0,0 +1,17 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param use_arena zero: false. non-zero: true.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h b/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.hnew file mode 100644index 0000000..8f8219f--- /dev/null+++ b/include/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h@@ -0,0 +1,15 @@+// Copyright 2019 AMD AMDMIGraphX++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id);++#ifdef __cplusplus+}+#endif++diff --git a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.hnew file mode 100644index 0000000..1da88b1--- /dev/null+++ b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h@@ -0,0 +1,35 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.+#pragma once++#include "onnxruntime_c_api.h"++// NNAPIFlags are bool options we want to set for NNAPI EP+// This enum is defined as bit flats, and cannot have negative value+// To generate an uint32_t nnapi_flags for using with OrtSessionOptionsAppendExecutionProvider_Nnapi below,+// uint32_t nnapi_flags = 0;+// nnapi_flags |= NNAPI_FLAG_USE_FP16;+enum NNAPIFlags {+ NNAPI_FLAG_USE_NONE = 0x000,++ // Using fp16 relaxation in NNAPI EP, this may improve perf but may also reduce precision+ NNAPI_FLAG_USE_FP16 = 0x001,++ // Use NCHW layout in NNAPI EP, this is only available after Android API level 29+ // Please note for now, NNAPI perform worse using NCHW compare to using NHWC+ NNAPI_FLAG_USE_NCHW = 0x002,++ // Keep NNAPI_FLAG_MAX at the end of the enum definition+ // And assign the last NNAPIFlag to it+ NNAPI_FLAG_LAST = NNAPI_FLAG_USE_NCHW,+};++#ifdef __cplusplus+extern "C" {+#endif++ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Nnapi, _In_ OrtSessionOptions* options, uint32_t nnapi_flags);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.h b/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.hnew file mode 100644index 0000000..ec574f7--- /dev/null+++ b/include/onnxruntime/core/providers/nuphar/nuphar_provider_factory.h@@ -0,0 +1,17 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.+#pragma once+#include "core/session/onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif+/**+ * \param device_id nuphar device id, starts from zero.+ * \param settings_str Nuphar settings string.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Nuphar, _In_ OrtSessionOptions* options, int allow_unaligned_buffers, _In_ const char* settings_str);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h b/include/onnxruntime/core/providers/openvino/openvino_provider_factory.hnew file mode 100644index 0000000..7646644--- /dev/null+++ b/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h@@ -0,0 +1,23 @@+// Copyright(C) 2019 Intel Corporation+// Licensed under the MIT License++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+struct ProviderInfo_OpenVINO {+ virtual std::vector<std::string> GetAvailableDevices() const = 0;+};++extern "C" {+#endif++/**+ * \param device_type openvino device type and precision. Could be any of+ * CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_OpenVINO,+ _In_ OrtSessionOptions* options, _In_ const char* device_type);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/providers.h b/include/onnxruntime/core/providers/providers.hnew file mode 100644index 0000000..2cfd5ac--- /dev/null+++ b/include/onnxruntime/core/providers/providers.h@@ -0,0 +1,13 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++namespace onnxruntime {+class IExecutionProvider;++struct IExecutionProviderFactory {+ virtual ~IExecutionProviderFactory() = default;+ virtual std::unique_ptr<IExecutionProvider> CreateProvider() = 0;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h b/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.hnew file mode 100644index 0000000..832171e--- /dev/null+++ b/include/onnxruntime/core/providers/rknpu/rknpu_provider_factory.h@@ -0,0 +1,14 @@+// Copyright 2020 rock-chips.com Inc.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Rknpu,+ _In_ OrtSessionOptions* options);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.hnew file mode 100644index 0000000..6152883--- /dev/null+++ b/include/onnxruntime/core/providers/rocm/rocm_provider_factory.h@@ -0,0 +1,17 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param device_id hip device id, starts from zero.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id, size_t hip_mem_limit);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.hnew file mode 100644index 0000000..44debc9--- /dev/null+++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h@@ -0,0 +1,14 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.hnew file mode 100644index 0000000..eb4765b--- /dev/null+++ b/include/onnxruntime/core/providers/vitisai/vitisai_provider_factory.h@@ -0,0 +1,17 @@+// Copyright (c) Xilinx Inc.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++#ifdef __cplusplus+extern "C" {+#endif++/**+ * \param use_arena zero: false. non-zero: true.+ */+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_VITISAI, _In_ OrtSessionOptions* options, const char *backend_type, int device_id);++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/providers/winml/winml_provider_factory.h b/include/onnxruntime/core/providers/winml/winml_provider_factory.hnew file mode 100644index 0000000..b08b42e--- /dev/null+++ b/include/onnxruntime/core/providers/winml/winml_provider_factory.h@@ -0,0 +1,9 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#include "onnxruntime_c_api.h"++struct WinmlAdapterApi;+typedef struct WinmlAdapterApi WinmlAdapterApi;++ORT_EXPORT const WinmlAdapterApi* ORT_API_CALL OrtGetWinMLAdapter(_In_ const OrtApi* ort_api) NO_EXCEPTION;\ No newline at end of filediff --git a/include/onnxruntime/core/session/automl_data_containers.h b/include/onnxruntime/core/session/automl_data_containers.hnew file mode 100644index 0000000..249e42a--- /dev/null+++ b/include/onnxruntime/core/session/automl_data_containers.h@@ -0,0 +1,30 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++// This header contains shared definitions for reading and writing+// via C/C++ API Opaque data types that are registered within ORT+#pragma once++#include <stdint.h>++#ifdef __cplusplus+extern "C" {+#endif+// This structure is used to initialize and read+// OrtValue of opaque(com.microsoft.featurizers,DateTimeFeaturizer_TimePoint)+struct DateTimeFeaturizerTimePointData {+ int32_t year;+ uint8_t month;+ uint8_t day;+ uint8_t hour;+ uint8_t minute;+ uint8_t second;+ uint8_t dayOfWeek;+ uint16_t dayOfYear;+ uint8_t quarterOfYear;+ uint8_t weekOfMonth;+};++#ifdef __cplusplus+} // extern "C"+#endifdiff --git a/include/onnxruntime/core/session/environment.h b/include/onnxruntime/core/session/environment.hnew file mode 100644index 0000000..5f7f6e2--- /dev/null+++ b/include/onnxruntime/core/session/environment.h@@ -0,0 +1,91 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++#include <atomic>+#include <memory>+#include "core/common/common.h"+#include "core/common/status.h"+#include "core/platform/threadpool.h"+#include "core/common/logging/logging.h"+#include "core/framework/allocator.h"++struct OrtThreadingOptions;+namespace onnxruntime {+/** TODO: remove this class+ Provides the runtime environment for onnxruntime.+ Create one instance for the duration of execution.+*/+class Environment {+ public:+ /**+ Create and initialize the runtime environment.+ @param logging manager instance that will enable per session logger output using+ session_options.session_logid as the logger id in messages.+ If nullptr, the default LoggingManager MUST have been created previously as it will be used+ for logging. This will use the default logger id in messages.+ See core/common/logging/logging.h for details, and how LoggingManager::DefaultLogger works.+ @param tp_options optional set of parameters controlling the number of intra and inter op threads for the global+ threadpools.+ @param create_global_thread_pools determine if this function will create the global threadpools or not.+ */+ static Status Create(std::unique_ptr<logging::LoggingManager> logging_manager,+ std::unique_ptr<Environment>& environment,+ const OrtThreadingOptions* tp_options = nullptr,+ bool create_global_thread_pools = false);++ logging::LoggingManager* GetLoggingManager() const {+ return logging_manager_.get();+ }++ void SetLoggingManager(std::unique_ptr<onnxruntime::logging::LoggingManager> logging_manager) {+ logging_manager_ = std::move(logging_manager);+ }++ onnxruntime::concurrency::ThreadPool* GetIntraOpThreadPool() const {+ return intra_op_thread_pool_.get();+ }++ onnxruntime::concurrency::ThreadPool* GetInterOpThreadPool() const {+ return inter_op_thread_pool_.get();+ }++ bool EnvCreatedWithGlobalThreadPools() const {+ return create_global_thread_pools_;+ }++ /**+ * Registers an allocator for sharing between multiple sessions.+ * Return an error if an allocator with the same OrtMemoryInfo is already registered.+ */+ Status RegisterAllocator(AllocatorPtr allocator);++ /**+ * Creates and registers an allocator for sharing between multiple sessions.+ * Return an error if an allocator with the same OrtMemoryInfo is already registered.+ */+ Status CreateAndRegisterAllocator(const OrtMemoryInfo& mem_info, const OrtArenaCfg* arena_cfg = nullptr);++ /**+ * Returns the list of registered allocators in this env.+ */+ const std::vector<AllocatorPtr>& GetRegisteredSharedAllocators() const {+ return shared_allocators_;+ }++ private:+ ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Environment);++ Environment() = default;+ Status Initialize(std::unique_ptr<logging::LoggingManager> logging_manager,+ const OrtThreadingOptions* tp_options = nullptr,+ bool create_global_thread_pools = false);++ std::unique_ptr<logging::LoggingManager> logging_manager_;+ std::unique_ptr<onnxruntime::concurrency::ThreadPool> intra_op_thread_pool_;+ std::unique_ptr<onnxruntime::concurrency::ThreadPool> inter_op_thread_pool_;+ bool create_global_thread_pools_{false};+ std::vector<AllocatorPtr> shared_allocators_;+};+} // namespace onnxruntimediff --git a/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.h b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.hnew file mode 100644index 0000000..6efb20a--- /dev/null+++ b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_api.h@@ -0,0 +1,69 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++// Summary: The experimental Ort C++ API is a wrapper around the Ort C++ API.+//+// This C++ API further simplifies usage and provides support for modern C++ syntax/features+// at the cost of some overhead (i.e. using std::string over char *).+//+// Where applicable, default memory allocator options are used unless explicitly set.+//+// Experimental components are designed as drop-in replacements of the regular API, requiring+// minimal code modifications to use.+//+// Example: Ort::Session -> Ort::Experimental::Session+//+// NOTE: Experimental API components are subject to change based on feedback and provide no+// guarantee of backwards compatibility in future releases.++#pragma once+#include "onnxruntime_cxx_api.h"++namespace Ort {+namespace Experimental {++struct Session : Ort::Session {+ Session(Env& env, std::basic_string<ORTCHAR_T>& model_path, SessionOptions& options)+ : Ort::Session(env, model_path.data(), options){};+ Session(Env& env, void* model_data, size_t model_data_length, SessionOptions& options)+ : Ort::Session(env, model_data, model_data_length, options){};++ // overloaded Run() with sensible defaults+ std::vector<Ort::Value> Run(const std::vector<std::string>& input_names,+ const std::vector<Ort::Value>& input_values,+ const std::vector<std::string>& output_names,+ const RunOptions& run_options = RunOptions());+ void Run(const std::vector<std::string>& input_names,+ const std::vector<Ort::Value>& input_values,+ const std::vector<std::string>& output_names,+ std::vector<Ort::Value>& output_values,+ const RunOptions& run_options = RunOptions());++ // convenience methods that simplify common lower-level API calls+ std::vector<std::string> GetInputNames() const;+ std::vector<std::string> GetOutputNames() const;+ std::vector<std::string> GetOverridableInitializerNames() const;++ // NOTE: shape dimensions may have a negative value to indicate a symbolic/unknown dimension.+ std::vector<std::vector<int64_t> > GetInputShapes() const;+ std::vector<std::vector<int64_t> > GetOutputShapes() const;+ std::vector<std::vector<int64_t> > GetOverridableInitializerShapes() const;+};++struct Value : Ort::Value {+ Value(OrtValue* p)+ : Ort::Value(p){};++ template <typename T>+ static Ort::Value CreateTensor(T* p_data, size_t p_data_element_count, const std::vector<int64_t>& shape);+ static Ort::Value CreateTensor(void* p_data, size_t p_data_byte_count, const std::vector<int64_t>& shape, ONNXTensorElementDataType type);++ template <typename T>+ static Ort::Value CreateTensor(const std::vector<int64_t>& shape);+ static Ort::Value CreateTensor(const std::vector<int64_t>& shape, ONNXTensorElementDataType type);+};++}+}++#include "experimental_onnxruntime_cxx_inline.h"diff --git a/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.hnew file mode 100644index 0000000..075de83--- /dev/null+++ b/include/onnxruntime/core/session/experimental_onnxruntime_cxx_inline.h@@ -0,0 +1,111 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++// Do not include this file directly. Please include "experimental_onnxruntime_cxx_api.h" instead.+//+// These are the inline implementations of the C++ header APIs. They are in this separate file as to not clutter+// the main C++ file with implementation details.+namespace Ort {+namespace Experimental {++inline std::vector<Ort::Value> Session::Run(const std::vector<std::string>& input_names, const std::vector<Ort::Value>& input_values,+ const std::vector<std::string>& output_names, const RunOptions& run_options) {+ size_t output_names_count = GetOutputNames().size();+ std::vector<Ort::Value> output_values;+ for (size_t i = 0; i < output_names_count; i++) output_values.emplace_back(nullptr);+ Run(input_names, input_values, output_names, output_values, run_options);+ return output_values;+}++inline void Session::Run(const std::vector<std::string>& input_names, const std::vector<Ort::Value>& input_values,+ const std::vector<std::string>& output_names, std::vector<Ort::Value>& output_values, const RunOptions& run_options) {+ size_t input_names_count = input_names.size();+ size_t output_names_count = output_names.size();+ std::vector<const char*> input_names_(input_names_count, nullptr);+ size_t i = 0;+ for (auto it = input_names.begin(); it != input_names.end(); it++) input_names_[i++] = (*it).c_str();+ std::vector<const char*> output_names_(output_names_count, nullptr);+ i = 0;+ for (auto it = output_names.begin(); it != output_names.end(); it++) output_names_[i++] = (*it).c_str();+ Ort::Session::Run(run_options, input_names_.data(), input_values.data(), input_names_count, output_names_.data(), output_values.data(), output_names_count);+}++inline std::vector<std::string> Session::GetInputNames() const {+ Ort::AllocatorWithDefaultOptions allocator;+ size_t node_count = GetInputCount();+ std::vector<std::string> out(node_count);+ for (size_t i = 0; i < node_count; i++) {+ char* tmp = GetInputName(i, allocator);+ out[i] = tmp;+ allocator.Free(tmp); // prevent memory leak+ }+ return out;+}++inline std::vector<std::string> Session::GetOutputNames() const {+ Ort::AllocatorWithDefaultOptions allocator;+ size_t node_count = GetOutputCount();+ std::vector<std::string> out(node_count);+ for (size_t i = 0; i < node_count; i++) {+ char* tmp = GetOutputName(i, allocator);+ out[i] = tmp;+ allocator.Free(tmp); // prevent memory leak+ }+ return out;+}++inline std::vector<std::string> Session::GetOverridableInitializerNames() const {+ Ort::AllocatorWithDefaultOptions allocator;+ size_t init_count = GetOverridableInitializerCount();+ std::vector<std::string> out(init_count);+ for (size_t i = 0; i < init_count; i++) {+ char* tmp = GetOverridableInitializerName(i, allocator);+ out[i] = tmp;+ allocator.Free(tmp); // prevent memory leak+ }+ return out;+}++inline std::vector<std::vector<int64_t>> Session::GetInputShapes() const {+ size_t node_count = GetInputCount();+ std::vector<std::vector<int64_t>> out(node_count);+ for (size_t i = 0; i < node_count; i++) out[i] = GetInputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ return out;+}++inline std::vector<std::vector<int64_t>> Session::GetOutputShapes() const {+ size_t node_count = GetOutputCount();+ std::vector<std::vector<int64_t>> out(node_count);+ for (size_t i = 0; i < node_count; i++) out[i] = GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ return out;+}++inline std::vector<std::vector<int64_t>> Session::GetOverridableInitializerShapes() const {+ size_t init_count = GetOverridableInitializerCount();+ std::vector<std::vector<int64_t>> out(init_count);+ for (size_t i = 0; i < init_count; i++) out[i] = GetOverridableInitializerTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ return out;+}++template <typename T>+inline Ort::Value Value::CreateTensor(T* p_data, size_t p_data_element_count, const std::vector<int64_t>& shape) {+ return CreateTensor(p_data, p_data_element_count * sizeof(T), shape, TypeToTensorType<T>::type);+}++inline Ort::Value Value::CreateTensor(void* p_data, size_t p_data_byte_count, const std::vector<int64_t>& shape, ONNXTensorElementDataType type) {+ Ort::MemoryInfo info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);+ return Ort::Value::CreateTensor(info, p_data, p_data_byte_count, shape.data(), shape.size(), type);+}++template <typename T>+inline Ort::Value Value::CreateTensor(const std::vector<int64_t>& shape) {+ return CreateTensor(shape, TypeToTensorType<T>::type);+}++inline Ort::Value Value::CreateTensor(const std::vector<int64_t>& shape, ONNXTensorElementDataType type) {+ Ort::AllocatorWithDefaultOptions allocator;+ return Ort::Value::CreateTensor(allocator, shape.data(), shape.size(), type);+}++}+}diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.hnew file mode 100644index 0000000..ab8b65c--- /dev/null+++ b/include/onnxruntime/core/session/onnxruntime_c_api.h@@ -0,0 +1,1193 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once+#include <stdlib.h>+#include <stdint.h>+#include <string.h>++// This value is used in structures passed to ORT so that a newer version of ORT will still work with them+#define ORT_API_VERSION 7++#ifdef __cplusplus+extern "C" {+#endif++// SAL2 Definitions+#ifndef _WIN32+#define _In_+#define _In_z_+#define _In_opt_+#define _In_opt_z_+#define _Out_+#define _Outptr_+#define _Out_opt_+#define _Inout_+#define _Inout_opt_+#define _Frees_ptr_opt_+#define _Ret_maybenull_+#define _Ret_notnull_+#define _Check_return_+#define _Outptr_result_maybenull_+#define _In_reads_(X)+#define _Inout_updates_all_(X)+#define _Out_writes_bytes_all_(X)+#define _Out_writes_all_(X)+#define _Success_(X)+#define _Outptr_result_buffer_maybenull_(X)+#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull))+#else+#include <specstrings.h>+#define ORT_ALL_ARGS_NONNULL+#endif++#ifdef _WIN32+// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort.+// dllexport is not used, we use a .def file.+#ifdef ORT_DLL_IMPORT+#define ORT_EXPORT __declspec(dllimport)+#else+#define ORT_EXPORT+#endif+#define ORT_API_CALL _stdcall+#define ORT_MUST_USE_RESULT+#define ORTCHAR_T wchar_t+#else+#define ORT_EXPORT+#define ORT_API_CALL+#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result))+#define ORTCHAR_T char+#endif++#ifndef ORT_TSTR+#ifdef _WIN32+#define ORT_TSTR(X) L##X+#else+#define ORT_TSTR(X) X+#endif+#endif++// Any pointer marked with _In_ or _Out_, cannot be NULL.++// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation+// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that.+// for ReleaseXXX(...) functions, they can accept NULL pointer.++#ifdef __cplusplus+// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept.+// Such complex condition is needed because compilers set __cplusplus value differently.+#ifndef __has_feature+#define __has_feature(x) 0+#endif+#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept)))+#define NO_EXCEPTION noexcept+#else+#define NO_EXCEPTION throw()+#endif+#else+#define NO_EXCEPTION+#endif++// Copied from TensorProto::DataType+// Currently, Ort doesn't support complex64, complex128+typedef enum ONNXTensorElementDataType {+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED,+ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string+ ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL,+ ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16,+ ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t+ ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components+ ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components+ ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision+} ONNXTensorElementDataType;++// Synced with onnx TypeProto oneof+typedef enum ONNXType {+ ONNX_TYPE_UNKNOWN,+ ONNX_TYPE_TENSOR,+ ONNX_TYPE_SEQUENCE,+ ONNX_TYPE_MAP,+ ONNX_TYPE_OPAQUE,+ ONNX_TYPE_SPARSETENSOR,+} ONNXType;++typedef enum OrtLoggingLevel {+ ORT_LOGGING_LEVEL_VERBOSE,+ ORT_LOGGING_LEVEL_INFO,+ ORT_LOGGING_LEVEL_WARNING,+ ORT_LOGGING_LEVEL_ERROR,+ ORT_LOGGING_LEVEL_FATAL,+} OrtLoggingLevel;++typedef enum OrtErrorCode {+ ORT_OK,+ ORT_FAIL,+ ORT_INVALID_ARGUMENT,+ ORT_NO_SUCHFILE,+ ORT_NO_MODEL,+ ORT_ENGINE_ERROR,+ ORT_RUNTIME_EXCEPTION,+ ORT_INVALID_PROTOBUF,+ ORT_MODEL_LOADED,+ ORT_NOT_IMPLEMENTED,+ ORT_INVALID_GRAPH,+ ORT_EP_FAIL,+} OrtErrorCode;++#define ORT_RUNTIME_CLASS(X) \+ struct Ort##X; \+ typedef struct Ort##X Ort##X;++// The actual types defined have an Ort prefix+ORT_RUNTIME_CLASS(Env);+ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success+ORT_RUNTIME_CLASS(MemoryInfo);+ORT_RUNTIME_CLASS(IoBinding);+ORT_RUNTIME_CLASS(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)+ORT_RUNTIME_CLASS(Value);+ORT_RUNTIME_CLASS(RunOptions);+ORT_RUNTIME_CLASS(TypeInfo);+ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);+ORT_RUNTIME_CLASS(SessionOptions);+ORT_RUNTIME_CLASS(CustomOpDomain);+ORT_RUNTIME_CLASS(MapTypeInfo);+ORT_RUNTIME_CLASS(SequenceTypeInfo);+ORT_RUNTIME_CLASS(ModelMetadata);+ORT_RUNTIME_CLASS(ThreadPoolParams);+ORT_RUNTIME_CLASS(ThreadingOptions);+ORT_RUNTIME_CLASS(ArenaCfg);++#ifdef _WIN32+typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;+#else+typedef OrtStatus* OrtStatusPtr;+#endif++// __VA_ARGS__ on Windows and Linux are different+#define ORT_API(RETURN_TYPE, NAME, ...) ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION++#define ORT_API_STATUS(NAME, ...) \+ ORT_EXPORT _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT++// XXX: Unfortunately, SAL annotations are known to not work with function pointers+#define ORT_API2_STATUS(NAME, ...) \+ _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT++// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT+#define ORT_API_STATUS_IMPL(NAME, ...) \+ _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION++#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input)++// When passing in an allocator to any ORT function, be sure that the allocator object+// is not destroyed until the last allocated object using it is freed.+typedef struct OrtAllocator {+ uint32_t version; // Initialize to ORT_API_VERSION+ void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size);+ void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p);+ const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_);+} OrtAllocator;++typedef void(ORT_API_CALL* OrtLoggingFunction)(+ void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location,+ const char* message);++// Set Graph optimization level.+// Refer https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md+// for in-depth undersrtanding of Graph Optimizations in ORT+typedef enum GraphOptimizationLevel {+ ORT_DISABLE_ALL = 0,+ ORT_ENABLE_BASIC = 1,+ ORT_ENABLE_EXTENDED = 2,+ ORT_ENABLE_ALL = 99+} GraphOptimizationLevel;++typedef enum ExecutionMode {+ ORT_SEQUENTIAL = 0,+ ORT_PARALLEL = 1,+} ExecutionMode;++// Set the language projection, default is C, which means it will classify the language not in the list to C also.+typedef enum OrtLanguageProjection {+ ORT_PROJECTION_C = 0, // default+ ORT_PROJECTION_CPLUSPLUS = 1,+ ORT_PROJECTION_CSHARP = 2,+ ORT_PROJECTION_PYTHON = 3,+ ORT_PROJECTION_JAVA = 4,+ ORT_PROJECTION_WINML = 5,+ ORT_PROJECTION_NODEJS = 6,+} OrtLanguageProjection;++struct OrtKernelInfo;+typedef struct OrtKernelInfo OrtKernelInfo;+struct OrtKernelContext;+typedef struct OrtKernelContext OrtKernelContext;+struct OrtCustomOp;+typedef struct OrtCustomOp OrtCustomOp;++typedef enum OrtAllocatorType {+ Invalid = -1,+ OrtDeviceAllocator = 0,+ OrtArenaAllocator = 1+} OrtAllocatorType;++/**+ * memory types for allocator, exec provider specific types should be extended in each provider+ * Whenever this struct is updated, please also update the MakeKey function in onnxruntime/core/framework/execution_provider.cc+*/+typedef enum OrtMemType {+ OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider+ OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED+ OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED+ OrtMemTypeDefault = 0, // the default allocator for execution provider+} OrtMemType;++typedef enum OrtCudnnConvAlgoSearch {+ EXHAUSTIVE, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx+ HEURISTIC, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7+ DEFAULT, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM+} OrtCudnnConvAlgoSearch;++/// <summary>+/// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA+/// </summary>+typedef struct OrtCUDAProviderOptions {+ int device_id; // cuda device with id=0 as default device.+ OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option+ size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t.+ int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo.+ int do_copy_in_default_stream;+} OrtCUDAProviderOptions;++/// <summary>+/// Options for the OpenVINO provider that are passed to SessionOptionsAppendExecutionProvider_OpenVINO+/// </summary>+typedef struct OrtOpenVINOProviderOptions {+#ifdef __cplusplus+ OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, num_of_threads{} {}+#endif+ const char* device_type; // CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32+ unsigned char enable_vpu_fast_compile; // 0 = false, nonzero = true+ const char* device_id;+ size_t num_of_threads; // 0 uses default number of threads+} OrtOpenVINOProviderOptions;++struct OrtApi;+typedef struct OrtApi OrtApi;++struct OrtApiBase {+ const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION+ // nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file++ const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION;+};+typedef struct OrtApiBase OrtApiBase;++ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION;++struct OrtApi {+ /**+* \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus+*/+ OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;++ OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;++ /**+ * \param status must not be NULL+ * \return The error message inside the `status`. Do not free the returned value.+ */+ const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;++ /**+ * \param out Should be freed by `OrtReleaseEnv` after use+ */+ ORT_API2_STATUS(CreateEnv, OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);++ /**+ * \param out Should be freed by `OrtReleaseEnv` after use+ */+ ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param,+ OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);++ // Platform telemetry events are on by default since they are lightweight. You can manually turn them off.+ ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env);+ ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env);++ // TODO: document the path separator convention? '/' vs '\'+ // TODO: should specify the access characteristics of model_path. Is this read only during the+ // execution of CreateSession, or does the OrtSession retain a handle to the file/directory+ // and continue to access throughout the OrtSession lifetime?+ // What sort of access is needed to model_path : read or read/write?+ ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,+ _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);++ ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length,+ _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);++ ORT_API2_STATUS(Run, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options,+ _In_reads_(input_len) const char* const* input_names,+ _In_reads_(input_len) const OrtValue* const* input, size_t input_len,+ _In_reads_(output_names_len) const char* const* output_names1, size_t output_names_len,+ _Inout_updates_all_(output_names_len) OrtValue** output);++ /**+ * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use+ */+ ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options);++ // Set filepath to save optimized model after graph level transformations.+ ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options,+ _In_ const ORTCHAR_T* optimized_model_filepath);++ // create a copy of an existing OrtSessionOptions+ ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options,+ _Outptr_ OrtSessionOptions** out_options);++ // Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model+ // has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance.+ // See [docs/ONNX_Runtime_Perf_Tuning.md] for more details.+ ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode);++ // Enable profiling for this session.+ ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix);+ ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options);++ // Enable the memory pattern optimization.+ // The idea is if the input shapes are the same, we could trace the internal memory allocation+ // and generate a memory pattern for future request. So next time we could just do one allocation+ // with a big chunk for all the internal memory allocation.+ // Note: memory pattern optimization is only available when SequentialExecution enabled.+ ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options);+ ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options);++ // Enable the memory arena on CPU+ // Arena may pre-allocate memory for future usage.+ // set this option to false if you don't want it.+ ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options);+ ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options);++ // < logger id to use for session output+ ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid);++ // < applies to session load, initialization, etc+ ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level);+ ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level);++ ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,+ GraphOptimizationLevel graph_optimization_level);++ // Sets the number of threads used to parallelize the execution within nodes+ // A value of 0 means ORT will pick a default+ // Note: If you've built ORT with OpenMP, this API has no effect on the number of threads used. In this case+ // use the OpenMP env variables to configure the number of intra op num threads.+ ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads);++ // Sets the number of threads used to parallelize the execution of the graph (across nodes)+ // If sequential execution is enabled this value is ignored+ // A value of 0 means ORT will pick a default+ ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads);++ /*+ Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain+ */+ ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out);++ /*+ * Add custom ops to the OrtCustomOpDomain+ * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released+ */+ ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op);++ /*+ * Add a custom op domain to the OrtSessionOptions+ * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released+ */+ ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain);++ /*+ * Loads a DLL named 'library_path' and looks for this entry point:+ * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api);+ * It then passes in the provided session options to this function along with the api base.+ * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in+ * session options are destroyed, or if an error occurs and it is non null.+ */+ ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path,+ void** library_handle);++ /**+ * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these+ * functions to enable them in the session:+ * OrtSessionOptionsAppendExecutionProvider_CPU+ * OrtSessionOptionsAppendExecutionProvider_CUDA+ * OrtSessionOptionsAppendExecutionProvider_<remaining providers...>+ * The order they are called indicates the preference order as well. In other words call this method+ * on your most preferred execution provider first followed by the less preferred ones.+ * If none are called Ort will use its internal CPU execution provider.+ */++ ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out);+ ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out);+ ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out);++ /**+ * \param out should be freed by OrtReleaseTypeInfo after use+ */+ ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info);++ /**+ * \param out should be freed by OrtReleaseTypeInfo after use+ */+ ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index,+ _Outptr_ OrtTypeInfo** type_info);++ /**+ * \param out should be freed by OrtReleaseTypeInfo after use+ */+ ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index,+ _Outptr_ OrtTypeInfo** type_info);++ /**+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.+ */+ ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,+ _Outptr_ char** value);+ ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,+ _Outptr_ char** value);+ ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index,+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);++ /**+ * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use+ */+ ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out);++ ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value);+ ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value);+ ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions*, _In_ const char* run_tag);++ ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);+ ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);+ ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out);++ // Set a flag so that ALL incomplete OrtRun calls that are using this instance of OrtRunOptions+ // will exit as soon as possible.+ ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options);+ // Unset the terminate flag to enable this OrtRunOptions instance being used in new OrtRun calls.+ ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options);++ /**+ * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value+ * \param out Should be freed by calling OrtReleaseValue+ * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx+ */+ ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len,+ ONNXTensorElementDataType type, _Outptr_ OrtValue** out);++ /**+ * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after.+ * p_data is owned by caller. OrtReleaseValue won't release p_data.+ * \param out Should be freed by calling OrtReleaseValue+ */+ ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data,+ size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type,+ _Outptr_ OrtValue** out);++ /**+ * \Sets *out to 1 iff an OrtValue is a tensor, 0 otherwise+ */+ ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out);++ // This function doesn't work with string tensor+ // this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.+ ORT_API2_STATUS(GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** out);++ /**+ * \param value A tensor created from OrtCreateTensor... function.+ * \param s each A string array. Each string in this array must be null terminated.+ * \param s_len length of s+ */+ ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len);++ /**+ * \param value A tensor created from OrtCreateTensor... function.+ * \param len total data length, not including the trailing '\0' chars.+ */+ ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len);++ /**+ * \param s string contents. Each string is NOT null-terminated.+ * \param value A tensor created from OrtCreateTensor... function.+ * \param s_len total data length, get it from OrtGetStringTensorDataLength+ */+ ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s,+ size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len);++ /**+ * Don't free the 'out' value+ */+ ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo*,+ _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out);++ /**+ * Return OnnxType from OrtTypeInfo+ */+ ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ enum ONNXType* out);++ /**+ * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo+ */+ ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out);++ ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type);++ /**+ * \param info Created from CreateTensorTypeAndShapeInfo() function+ * \param dim_values An array with length of `dim_count`. Its elements can contain negative values.+ * \param dim_count length of dim_values+ */+ ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);++ ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*,+ _Out_ enum ONNXTensorElementDataType* out);+ ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);+ ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values,+ size_t dim_values_length);+ ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info,+ _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length);++ /**+ * Return the number of elements specified by the tensor shape.+ * Return a negative value if unknown (i.e., any dimension is negative.)+ * e.g.+ * [] -> 1+ * [1,3,4] -> 12+ * [2,0,4] -> 0+ * [-1,3,4] -> -1+ */+ ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);++ /**+ * \param out Should be freed by OrtReleaseTensorTypeAndShapeInfo after use+ */+ ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out);++ /**+ * Get the type information of an OrtValue+ * \param value+ * \param out The returned value should be freed by OrtReleaseTypeInfo after use+ */+ ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out);++ ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out);++ ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1,+ enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out);++ /**+ * Convenience function for special case of CreateMemoryInfo, for the CPU allocator. Uses name = "Cpu" and id = 0.+ */+ ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1,+ _Outptr_ OrtMemoryInfo** out);++ /**+ * Test if two memory info are equal+ * \Sets 'out' to 0 if equal, -1 if not equal+ */+ ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out);++ /**+ * Do not free the returned value+ */+ ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out);+ ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out);+ ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out);+ ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out);++ ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outptr_ void** out);+ ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ptr, void* p);+ ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ const struct OrtMemoryInfo** out);++ // The returned pointer doesn't have to be freed.+ // Always returns the same instance on every invocation.+ // Please note that this is a non-arena based allocator.+ ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out);++ // Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable+ // optimizations that can take advantage of fixed values (such as memory planning, etc)+ ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation,+ _In_ int64_t dim_value);++ /**+ * APIs to support non-tensor types - map and sequence.+ * Currently only the following types are supported+ * Note: the following types should be kept in sync with data_types.h+ * Map types+ * =========+ * std::map<std::string, std::string>+ * std::map<std::string, int64_t>+ * std::map<std::string, float>+ * std::map<std::string, double>+ * std::map<int64_t, std::string>+ * std::map<int64_t, int64_t>+ * std::map<int64_t, float>+ * std::map<int64_t, double>+ *+ * Sequence types+ * ==============+ * std::vector<std::string>+ * std::vector<int64_t>+ * std::vector<float>+ * std::vector<double>+ * std::vector<std::map<std::string, float>>+ * std::vector<std::map<int64_t, float>+ */++ /**+ * If input OrtValue represents a map, you need to retrieve the keys and values+ * separately. Use index=0 to retrieve keys and index=1 to retrieve values.+ * If input OrtValue represents a sequence, use index to retrieve the index'th element+ * of the sequence.+ */+ ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator,+ _Outptr_ OrtValue** out);++ /**+ * Returns 2 for type map and N for sequence where N is the number of elements+ * in the sequence.+ */+ ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out);++ /**+ * To construct a map, use num_values = 2 and 'in' should be an arrary of 2 OrtValues+ * representing keys and values.+ * To construct a sequence, use num_values = N where N is the number of the elements in the+ * sequence. 'in' should be an arrary of N OrtValues.+ * \value_type should be either map or sequence.+ */+ ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values,+ enum ONNXType value_type, _Outptr_ OrtValue** out);++ /**+ * Construct OrtValue that contains a value of non-standard type created for+ * experiments or while awaiting standardization. OrtValue in this case would contain+ * an internal representation of the Opaque type. Opaque types are distinguished between+ * each other by two strings 1) domain and 2) type name. The combination of the two+ * must be unique, so the type representation is properly identified internally. The combination+ * must be properly registered from within ORT at both compile/run time or by another API.+ *+ * To construct the OrtValue pass domain and type names, also a pointer to a data container+ * the type of which must be know to both ORT and the client program. That data container may or may+ * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for+ * verification purposes.+ *+ * \domain_name - domain name for the Opaque type, null terminated.+ * \type_name - type name for the Opaque type, null terminated.+ * \data_contianer - data to populate OrtValue+ * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected+ * data_container size internally.+ */+ ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name,+ _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out);++ /**+ * Fetch data from an OrtValue that contains a value of non-standard type created for+ * experiments or while awaiting standardization.+ * \domain_name - domain name for the Opaque type, null terminated.+ * \type_name - type name for the Opaque type, null terminated.+ * \data_contianer - data to populate OrtValue+ * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected+ * data_container size internally.+ */++ ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in,+ _Out_ void* data_container, size_t data_container_size);++ ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name,+ _Out_ float* out);+ ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name,+ _Out_ int64_t* out);+ ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out,+ _Inout_ size_t* size);++ ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);+ ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);+ ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index,+ _Out_ const OrtValue** out);+ ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index,+ _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out);++ ORT_CLASS_RELEASE(Env);+ ORT_CLASS_RELEASE(Status); // nullptr for Status* indicates success+ ORT_CLASS_RELEASE(MemoryInfo);+ ORT_CLASS_RELEASE(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)+ ORT_CLASS_RELEASE(Value);+ ORT_CLASS_RELEASE(RunOptions);+ ORT_CLASS_RELEASE(TypeInfo);+ ORT_CLASS_RELEASE(TensorTypeAndShapeInfo);+ ORT_CLASS_RELEASE(SessionOptions);+ ORT_CLASS_RELEASE(CustomOpDomain);++ // End of Version 1 - DO NOT MODIFY ABOVE (see above text for more information)++ // Version 2 - In development, feel free to add/remove/rearrange here++ /**+ * GetDenotationFromTypeInfo+ * This api augments OrtTypeInfo to return denotations on the type.+ * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor.+ */+ ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ const char** const denotation,+ _Out_ size_t* len);++ // OrtTypeInfo Casting methods++ /**+ * CastTypeInfoToMapTypeInfo+ * This api augments OrtTypeInfo to return an OrtMapTypeInfo when the type is a map.+ * The OrtMapTypeInfo has additional information about the map's key type and value type.+ * This is used by WinML to support model reflection APIs.+ * This is used by WinML to support model reflection APIs.+ *+ * Don't free the 'out' value+ */+ ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info,+ _Outptr_result_maybenull_ const OrtMapTypeInfo** out);++ /**+ * CastTypeInfoToSequenceTypeInfo+ * This api augments OrtTypeInfo to return an OrtSequenceTypeInfo when the type is a sequence.+ * The OrtSequenceTypeInfo has additional information about the sequence's element type.+ * This is used by WinML to support model reflection APIs.+ *+ * Don't free the 'out' value+ */+ ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info,+ _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out);++ // OrtMapTypeInfo Accessors++ /**+ * GetMapKeyType+ * This api augments get the key type of a map. Key types are restricted to being scalar types and use ONNXTensorElementDataType.+ * This is used by WinML to support model reflection APIs.+ */+ ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out);++ /**+ * GetMapValueType+ * This api augments get the value type of a map.+ */+ ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info);++ // OrtSequenceTypeInfo Accessors++ /**+ * GetSequenceElementType+ * This api augments get the element type of a sequence.+ * This is used by WinML to support model reflection APIs.+ */+ ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info,+ _Outptr_ OrtTypeInfo** type_info);++ ORT_CLASS_RELEASE(MapTypeInfo);+ ORT_CLASS_RELEASE(SequenceTypeInfo);++ /**+ * \param out is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.+ * Profiling is turned ON automatically if enabled for the particular session by invoking EnableProfiling()+ * on the SessionOptions instance used to create the session.+ */+ ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* sess, _Inout_ OrtAllocator* allocator, _Outptr_ char** out);++ /**+ * \param out is a pointer to the newly created object. The pointer should be freed by calling ReleaseModelMetadata after use.+ */+ ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* sess, _Outptr_ OrtModelMetadata** out);++ /**+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.+ */+ ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);+ ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);+ ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator,+ _Outptr_ char** value);+ ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);+ /**+ * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.+ * 'value' will be a nullptr if the given key is not found in the custom metadata map.+ */+ ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value);++ ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value);++ ORT_CLASS_RELEASE(ModelMetadata);++ /*+ * Creates an environment with global threadpools that will be shared across sessions.+ * Use this in conjunction with DisablePerSessionThreads API or else the session will use+ * its own thread pools.+ */+ ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel logging_level, _In_ const char* logid,+ _In_ const OrtThreadingOptions* t_options, _Outptr_ OrtEnv** out);++ /*+ * Calling this API will make the session use the global threadpools shared across sessions.+ * This API should be used in conjunction with CreateEnvWithGlobalThreadPools API.+ */+ ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options);++ ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out);++ ORT_CLASS_RELEASE(ThreadingOptions);++ /**+ * \param num_keys contains the number of keys in the custom metadata map+ * \param keys is an array of null terminated strings (array count = num_keys) allocated using 'allocator'.+ * The caller is responsible for freeing each string and the pointer array.+ * 'keys' will be a nullptr if custom metadata map is empty.+ */+ ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys);++ // Override symbolic dimensions (by specific name strings) with actual values+ // if known at session initialization time to enable optimizations that can+ // take advantage of fixed values (such as memory planning, etc)+ ORT_API2_STATUS(AddFreeDimensionOverrideByName,+ _Inout_ OrtSessionOptions* options, _In_ const char* dim_name,+ _In_ int64_t dim_value);++ /**+ * \param out_ptr will hold a pointer to the array of char *+ * representing available providers.+ * \param provider_length is a pointer to an int variable where+ * the number of available providers will be added.+ * The caller is responsible for freeing each char * and the pointer+ * array by calling ReleaseAvailableProviders().+ */+ ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr,+ _In_ int* provider_length);++ /**+ * \param ptr is the pointer to an array of available providers you+ * get after calling GetAvailableProviders().+ * \param providers_length is the number of available providers.+ */+ ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr,+ _In_ int providers_length);++ /**+ * \param value - A tensor created from OrtCreateTensor... function.+ * \param index - index of string tensor element, length of element at index will be returned.+ * \param out - number of UTF-8 bytes that the string contains+ */+ ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out);++ /**+ * \param s string element contents in UTF-8 encoding. The string is NOT null-terminated.+ * \param value A tensor created from OrtCreateTensor... function.+ * \param s_len element length, get it from OrtGetStringTensorElementLength.+ * \param index offset of element of tensor to return.+ */+ ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s);++ /**+ * \param value - A tensor created from OrtCreateTensor... function.+ * \param s - A null terminated UTF-8 encoded string.+ * \param index - index of string tensor element to fill+ */+ ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index);++ /**+ * Set a single session configuration entry as a pair of strings+ * If a configuration with same key exists, this will overwrite the configuration with the given config_value+ * \param config_key A null terminated string representation of the config key+ * \param config_value A null terminated string representation of the config value+ * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h+ */+ ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options,+ _In_z_ const char* config_key, _In_z_ const char* config_value);++ /**+ * \param sess valid OrtSession instance+ * \param mem_info - valid OrtMemoryInfo instance+ * \param - out a ptr to a new instance of OrtAllocator according to the spec within mem_info+ * if successful+ * \return OrtStatus or nullptr if successful+ */+ ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* sess, _In_ const OrtMemoryInfo* mem_info,+ _Outptr_ OrtAllocator** out);++ // Release instance of OrtAllocator obtained from CreateAllocator API+ ORT_CLASS_RELEASE(Allocator);++ ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr);++ // Creates an IoBinding instance that allows one to bind pre-allocated OrtValues+ // to input names. Thus if you want to use a raw on device buffer as input or output+ // you can avoid extra copy during runtime.+ ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* sess, _Outptr_ OrtIoBinding** out);++ // Release instance or OrtIoBinding obtained from CreateIoBinding API+ ORT_CLASS_RELEASE(IoBinding);++ /**+ * The function will bind the OrtValue to a specified input name.+ * The OrtValue must be a Tensor. ORT would use that value in place of input for the specified name.+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()+ * \param name - name for the model input+ * \param val_ptr - OrtValue of Tensor type.+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success+ */+ ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);++ /**+ * The function will bind the OrtValue to the specified output name.+ * The OrtValue must be a Tensor. ORT would use that value in place of output for the specified name.+ *+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()+ * \param name - name for the model output+ * \param val_ptr - OrtValue of Tensor type.+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success+ */+ ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);++ /**+ * The function will bind the OrtValue to a device which specification is contained within OrtMemoryInfo+ * You can either create an instance of OrtMemoryInfo with a device id or obtain one from the allocator that you are created/using+ * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocated and bind a chunk of+ * memory within OrtValue ahead of time.+ *+ * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()+ * \param name - name for the model output+ * \param mem_info_ptr - OrtMemoryInfo+ * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success+ */+ ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* val_ptr);++ /**+ * The function returns the names of the outputs in the order they were bound. This is useful after running the model+ * with bound outputs because the returned names are in order in which output OrtValues are returned. This API is optional+ * to use. If you knew the order of outputs and its names you used for binding you would not need to use this API.+ *+ * \param binding_ptr - a ptr to an instance of OrtIoBinding created obtained from CreateIoBinding()+ * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions()+ * the specified allocator will be used to allocate continuous buffers for output strings and lengths.+ * \param buffer - pointer to a continuous buffer of non-zero terminated UTF-8 encoded strings. The number of strings stored is returned count parameter.+ * this buffer will be allocated with the specified allocator and must be freed after it is no longer needed.+ * \param lengths - a pointer to a continuous buffer of size_t lengths of strings returned in the buffer. The number of items is returned+ * in the count. This buffer is allocated with the specified allocator and must be freed after it is no longer needed.+ * \para count - is the number of strings returned. If the instance of OrtIoBiding has no bound outputs, zero is returned,+ * no memory allocation is performed and buffer and lengths are nullptr on return.+ */+ ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,+ _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count);++ /**+ * The function returns an array of pointers to individually allocated OrtValues that contain results of a model execution with RunWithBinding()+ * The array contains the same number of OrtValues and they are in the same order as they were bound with BindOutput()+ * or BindOutputToDevice().+ * The returned OrtValues must be individually released after they are no longer needed.+ * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after+ * all the OrtValues contained therein are individually released.+ *+ * \param binding_ptr - instance of OrtIoBidning+ * \param allocator - instance of allocator to allocate output array+ * \param output - pointer to the allocated buffer. Returns nullptr if no outputs.+ * \param output_count - pointer to the number of OrtValues returned. Zero if no outputs.+ */+ ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,+ _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count);++ /** Clears any previously specified bindings for inputs/outputs+ */+ void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;+ void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;++ /**+ * Provides element-level access into a tensor.+ * \param location_values a pointer to an array of index values that specify an element's location in the tensor data blob+ * \param location_values_count length of location_values+ * \param out a pointer to the element specified by location_values+ * e.g.+ * Given a tensor with overall shape [3,224,224], an element at+ * location [2,150,128] can be accessed directly.+ *+ * This function only works for numeric tensors.+ * This is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.+ */+ ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out);++ /**+ * Creates an allocator instance and registers it with the env to enable+ * sharing between multiple sessions that use the same env instance.+ * Lifetime of the created allocator will be valid for the duration of the environment.+ * Returns an error if an allocator with the same OrtMemoryInfo is already registered.+ * \param mem_info must be non-null.+ * \param arena_cfg if nullptr defaults will be used.+ * See docs/C_API.md for details.+ */+ ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info,+ _In_ const OrtArenaCfg* arena_cfg);++ /**+ * Set the language projection for collecting telemetry data when Env is created+ * \param projection the source projected language.+ */+ ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection);++ /**+ * On some platforms, this timer may not be as precise as nanoseconds+ * For instance, on Windows and MacOS, the precision will be ~100ns+ * \param out is set to the nanoseconds of profiling's start time+ */+ ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* sess, _Outptr_ uint64_t* out);++ /**+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.+ * A value of 0 means ORT will pick the default.+ * A value of 1 means the invoking thread will be used; no threads will be created in the thread pool.+ */+ ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads);+ ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads);++ /**+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.+ * Allow spinning of thread pools when their queues are empty. This API will set the value for both+ * inter_op and intra_op threadpools.+ * \param allow_spinning valid values are 1 and 0.+ * 1: threadpool will spin to wait for queue to become non-empty, 0: it won't spin.+ * Prefer a value of 0 if your CPU usage is very high.+ */+ ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning);++ /**+ * Add a pre-allocated initializer to a session. If a model contains an initializer with a name+ * that is same as the name passed to this API call, ORT will use this initializer instance+ * instead of deserializing one from the model file. This is useful when you want to share+ * the same initializer across sessions.+ * \param name name of the initializer+ * \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be+ * managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object+ * to which it is added.+ */+ ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name,+ _In_ const OrtValue* val);++ /**+ * Creates a custom environment with global threadpools and logger that will be shared across sessions.+ * Use this in conjunction with DisablePerSessionThreads API or else the session will use+ * its own thread pools.+ *+ * \param out should be freed by `OrtReleaseEnv` after use+ */+ ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel logging_level,+ _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out);++ /**+ * Append CUDA execution provider to the session options+ * If CUDA is not available (due to a non cuda enabled build), this function will return failure.+ */+ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA,+ _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options);++ /**+ * Append OpenVINO execution provider to the session options+ * If OpenVINO is not available (due to the OpenVINO provider shared library or its dependencies not being installed), this function will fail.+ */+ ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO,+ _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options);++ /**+ * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.+ * When this API is called, flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool.+ * Note that an alternative way not using this option at runtime is to train and export a model without denormals+ * and that's recommended because turning this option on may hurt model accuracy.+ */+ ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options);++ /**+ * Use this API to create the configuration of an arena that can eventually be used to define+ * an arena based allocator's behavior+ * \param max_mem - use 0 to allow ORT to choose the default+ * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested+ * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default+ * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default+ * \param out - a pointer to an OrtArenaCfg instance+ * \return a nullptr in case of success or a pointer to an OrtStatus instance in case of failure+ * See docs/C_API.md for details on what the following parameters mean and how to choose these values+ */+ ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes,+ int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out);++ ORT_CLASS_RELEASE(ArenaCfg);++ /**+ * Use this API to obtain the description of the graph present in the model+ * (doc_string field of the GraphProto message within the ModelProto message).+ * If it doesn't exist, an empty string will be returned.+ * \param model_metadata - an instance of OrtModelMetadata+ * \param allocator - allocator used to allocate the string that will be returned back+ * \param value - is set to a null terminated string allocated using 'allocator'.+ The caller is responsible for freeing it.+ */+ ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata,+ _Inout_ OrtAllocator* allocator, _Outptr_ char** value);+};++/*+ * Steps to use a custom op:+ * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops+ * 2 Create an OrtCustomOp structure for each op and add them to the domain+ * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options+*/+#define OrtCustomOpApi OrtApi++/*+ * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by+ * the implementor of the custom op.+*/+struct OrtCustomOp {+ uint32_t version; // Initialize to ORT_API_VERSION++ // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below.+ void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api,+ _In_ const OrtKernelInfo* info);++ // Returns the name of the op+ const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op);++ // Returns the type of the execution provider, return nullptr to use CPU execution provider+ const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op);++ // Returns the count and types of the input & output tensors+ ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);+ size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op);+ ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);+ size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op);++ // Op kernel callbacks+ void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context);+ void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel);+};++/*+ * END EXPERIMENTAL+*/++#ifdef __cplusplus+}+#endifdiff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.hnew file mode 100644index 0000000..d5aa79a--- /dev/null+++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h@@ -0,0 +1,633 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++// Summary: The Ort C++ API is a header only wrapper around the Ort C API.+//+// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors+// and automatically releasing resources in the destructors.+//+// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.+// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};).+//+// Only move assignment between objects is allowed, there are no copy constructors. Some objects have explicit 'Clone'+// methods for this purpose.++#pragma once+#include "onnxruntime_c_api.h"+#include <cstddef>+#include <array>+#include <memory>+#include <stdexcept>+#include <string>+#include <vector>+#include <utility>+#include <type_traits>++#ifdef ORT_NO_EXCEPTIONS+#include <iostream>+#endif++namespace Ort {++// All C++ methods that can fail will throw an exception of this type+struct Exception : std::exception {+ Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}++ OrtErrorCode GetOrtErrorCode() const { return code_; }+ const char* what() const noexcept override { return message_.c_str(); }++ private:+ std::string message_;+ OrtErrorCode code_;+};++#ifdef ORT_NO_EXCEPTIONS+#define ORT_CXX_API_THROW(string, code) \+ do { \+ std::cerr << Ort::Exception(string, code) \+ .what() \+ << std::endl; \+ abort(); \+ } while (false)+#else+#define ORT_CXX_API_THROW(string, code) \+ throw Ort::Exception(string, code)+#endif++// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, it's in a template so that we can define a global variable in a header and make+// it transparent to the users of the API.+template <typename T>+struct Global {+ static const OrtApi* api_;+};++// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.++template <typename T>+#ifdef ORT_API_MANUAL_INIT+const OrtApi* Global<T>::api_{};+inline void InitApi() { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }+#else+const OrtApi* Global<T>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION);+#endif++// This returns a reference to the OrtApi interface in use, in case someone wants to use the C API functions+inline const OrtApi& GetApi() { return *Global<void>::api_; }++// This is a C++ wrapper for GetAvailableProviders() C API and returns+// a vector of strings representing the available execution providers.+std::vector<std::string> GetAvailableProviders();++// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type+// This can't be done in the C API since C doesn't have function overloading.+#define ORT_DEFINE_RELEASE(NAME) \+ inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }++ORT_DEFINE_RELEASE(Allocator);+ORT_DEFINE_RELEASE(MemoryInfo);+ORT_DEFINE_RELEASE(CustomOpDomain);+ORT_DEFINE_RELEASE(Env);+ORT_DEFINE_RELEASE(RunOptions);+ORT_DEFINE_RELEASE(Session);+ORT_DEFINE_RELEASE(SessionOptions);+ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);+ORT_DEFINE_RELEASE(SequenceTypeInfo);+ORT_DEFINE_RELEASE(MapTypeInfo);+ORT_DEFINE_RELEASE(TypeInfo);+ORT_DEFINE_RELEASE(Value);+ORT_DEFINE_RELEASE(ModelMetadata);+ORT_DEFINE_RELEASE(ThreadingOptions);+ORT_DEFINE_RELEASE(IoBinding);+ORT_DEFINE_RELEASE(ArenaCfg);++/*! \class Ort::Float16_t+ * \brief it is a structure that represents float16 data.+ * \details It is necessary for type dispatching to make use of C++ API+ * The type is implicitly convertible to/from uint16_t.+ * The size of the structure should align with uint16_t and one can freely cast+ * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data.+ *+ * Generally, you can feed any of your types as float16/blfoat16 data to create a tensor+ * on top of it, providing it can form a continuous buffer with 16-bit elements with no padding.+ * And you can also feed a array of uint16_t elements directly. For example,+ *+ * \code{.unparsed}+ * uint16_t values[] = { 15360, 16384, 16896, 17408, 17664};+ * constexpr size_t values_length = sizeof(values) / sizeof(values[0]);+ * std::vector<int64_t> dims = {values_length}; // one dimensional example+ * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);+ * // Note we are passing bytes count in this api, not number of elements -> sizeof(values)+ * auto float16_tensor = Ort::Value::CreateTensor(info, values, sizeof(values),+ * dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16);+ * \endcode+ *+ * Here is another example, a little bit more elaborate. Let's assume that you use your own float16 type and you want to use+ * a templated version of the API above so the type is automatically set based on your type. You will need to supply an extra+ * template specialization.+ *+ * \code{.unparsed}+ * namespace yours { struct half {}; } // assume this is your type, define this:+ * namespace Ort {+ * template<>+ * struct TypeToTensorType<yours::half> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; };+ * } //namespace Ort+ *+ * std::vector<yours::half> values;+ * std::vector<int64_t> dims = {values.size()}; // one dimensional example+ * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault);+ * // Here we are passing element count -> values.size()+ * auto float16_tensor = Ort::Value::CreateTensor<yours::half>(info, values.data(), values.size(), dims.data(), dims.size());+ *+ * \endcode+ */+struct Float16_t {+ uint16_t value;+ constexpr Float16_t() noexcept : value(0) {}+ constexpr Float16_t(uint16_t v) noexcept : value(v) {}+ constexpr operator uint16_t() const noexcept { return value; }+ constexpr bool operator==(const Float16_t& rhs) const noexcept { return value == rhs.value; };+ constexpr bool operator!=(const Float16_t& rhs) const noexcept { return value != rhs.value; };+};++static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");++/*! \class Ort::BFloat16_t+ * \brief is a structure that represents bfloat16 data.+ * \details It is necessary for type dispatching to make use of C++ API+ * The type is implicitly convertible to/from uint16_t.+ * The size of the structure should align with uint16_t and one can freely cast+ * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data.+ *+ * See also code examples for Float16_t above.+ */+struct BFloat16_t {+ uint16_t value;+ constexpr BFloat16_t() noexcept : value(0) {}+ constexpr BFloat16_t(uint16_t v) noexcept : value(v) {}+ constexpr operator uint16_t() const noexcept { return value; }+ constexpr bool operator==(const BFloat16_t& rhs) const noexcept { return value == rhs.value; };+ constexpr bool operator!=(const BFloat16_t& rhs) const noexcept { return value != rhs.value; };+};++static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");++// This is used internally by the C++ API. This is the common base class used by the wrapper objects.+template <typename T>+struct Base {+ using contained_type = T;++ Base() = default;+ Base(T* p) : p_{p} {+ if (!p)+ ORT_CXX_API_THROW("Allocation failure", ORT_FAIL);+ }+ ~Base() { OrtRelease(p_); }++ operator T*() { return p_; }+ operator const T*() const { return p_; }++ T* release() {+ T* p = p_;+ p_ = nullptr;+ return p;+ }++ protected:+ Base(const Base&) = delete;+ Base& operator=(const Base&) = delete;+ Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }+ void operator=(Base&& v) noexcept {+ OrtRelease(p_);+ p_ = v.p_;+ v.p_ = nullptr;+ }++ T* p_{};++ template <typename>+ friend struct Unowned; // This friend line is needed to keep the centos C++ compiler from giving an error+};++template <typename T>+struct Base<const T> {+ using contained_type = const T;++ Base() = default;+ Base(const T* p) : p_{p} {+ if (!p)+ ORT_CXX_API_THROW("Invalid instance ptr", ORT_INVALID_ARGUMENT);+ }+ ~Base() = default;++ operator const T*() const { return p_; }++ protected:+ Base(const Base&) = delete;+ Base& operator=(const Base&) = delete;+ Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }+ void operator=(Base&& v) noexcept {+ p_ = v.p_;+ v.p_ = nullptr;+ }++ const T* p_{};+};++template <typename T>+struct Unowned : T {+ Unowned(decltype(T::p_) p) : T{p} {}+ Unowned(Unowned&& v) : T{v.p_} {}+ ~Unowned() { this->release(); }+};++struct AllocatorWithDefaultOptions;+struct MemoryInfo;+struct Env;+struct TypeInfo;+struct Value;+struct ModelMetadata;++struct Env : Base<OrtEnv> {+ Env(std::nullptr_t) {}+ Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");+ Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");+ Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);+ Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,+ OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");+ explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}++ Env& EnableTelemetryEvents();+ Env& DisableTelemetryEvents();++ Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);++ static const OrtApi* s_api;+};++struct CustomOpDomain : Base<OrtCustomOpDomain> {+ explicit CustomOpDomain(std::nullptr_t) {}+ explicit CustomOpDomain(const char* domain);++ void Add(OrtCustomOp* op);+};++struct RunOptions : Base<OrtRunOptions> {+ RunOptions(std::nullptr_t) {}+ RunOptions();++ RunOptions& SetRunLogVerbosityLevel(int);+ int GetRunLogVerbosityLevel() const;++ RunOptions& SetRunLogSeverityLevel(int);+ int GetRunLogSeverityLevel() const;++ RunOptions& SetRunTag(const char* run_tag);+ const char* GetRunTag() const;++ // terminate ALL currently executing Session::Run calls that were made using this RunOptions instance+ RunOptions& SetTerminate();+ // unset the terminate flag so this RunOptions instance can be used in a new Session::Run call+ RunOptions& UnsetTerminate();+};++struct SessionOptions : Base<OrtSessionOptions> {+ explicit SessionOptions(std::nullptr_t) {}+ SessionOptions();+ explicit SessionOptions(OrtSessionOptions* p) : Base<OrtSessionOptions>{p} {}++ SessionOptions Clone() const;++ SessionOptions& SetIntraOpNumThreads(int intra_op_num_threads);+ SessionOptions& SetInterOpNumThreads(int inter_op_num_threads);+ SessionOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);++ SessionOptions& EnableCpuMemArena();+ SessionOptions& DisableCpuMemArena();++ SessionOptions& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);++ SessionOptions& EnableProfiling(const ORTCHAR_T* profile_file_prefix);+ SessionOptions& DisableProfiling();++ SessionOptions& EnableMemPattern();+ SessionOptions& DisableMemPattern();++ SessionOptions& SetExecutionMode(ExecutionMode execution_mode);++ SessionOptions& SetLogId(const char* logid);+ SessionOptions& SetLogSeverityLevel(int level);++ SessionOptions& Add(OrtCustomOpDomain* custom_op_domain);++ SessionOptions& DisablePerSessionThreads();++ SessionOptions& AddConfigEntry(const char* config_key, const char* config_value);+ SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val);++ SessionOptions& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);+ SessionOptions& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);+};++struct ModelMetadata : Base<OrtModelMetadata> {+ explicit ModelMetadata(std::nullptr_t) {}+ explicit ModelMetadata(OrtModelMetadata* p) : Base<OrtModelMetadata>{p} {}++ char* GetProducerName(OrtAllocator* allocator) const;+ char* GetGraphName(OrtAllocator* allocator) const;+ char* GetDomain(OrtAllocator* allocator) const;+ char* GetDescription(OrtAllocator* allocator) const;+ char* GetGraphDescription(OrtAllocator* allocator) const;+ char** GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const;+ char* LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const;+ int64_t GetVersion() const;+};++struct Session : Base<OrtSession> {+ explicit Session(std::nullptr_t) {}+ Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);+ Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);++ // Run that will allocate the output values+ std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,+ const char* const* output_names, size_t output_count);+ // Run for when there is a list of preallocated outputs+ void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,+ const char* const* output_names, Value* output_values, size_t output_count);++ void Run(const RunOptions& run_options, const struct IoBinding&);++ size_t GetInputCount() const;+ size_t GetOutputCount() const;+ size_t GetOverridableInitializerCount() const;++ char* GetInputName(size_t index, OrtAllocator* allocator) const;+ char* GetOutputName(size_t index, OrtAllocator* allocator) const;+ char* GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const;+ char* EndProfiling(OrtAllocator* allocator) const;+ uint64_t GetProfilingStartTimeNs() const;+ ModelMetadata GetModelMetadata() const;++ TypeInfo GetInputTypeInfo(size_t index) const;+ TypeInfo GetOutputTypeInfo(size_t index) const;+ TypeInfo GetOverridableInitializerTypeInfo(size_t index) const;+};++struct TensorTypeAndShapeInfo : Base<OrtTensorTypeAndShapeInfo> {+ explicit TensorTypeAndShapeInfo(std::nullptr_t) {}+ explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : Base<OrtTensorTypeAndShapeInfo>{p} {}++ ONNXTensorElementDataType GetElementType() const;+ size_t GetElementCount() const;++ size_t GetDimensionsCount() const;+ void GetDimensions(int64_t* values, size_t values_count) const;+ void GetSymbolicDimensions(const char** values, size_t values_count) const;++ std::vector<int64_t> GetShape() const;+};++struct SequenceTypeInfo : Base<OrtSequenceTypeInfo> {+ explicit SequenceTypeInfo(std::nullptr_t) {}+ explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : Base<OrtSequenceTypeInfo>{p} {}++ TypeInfo GetSequenceElementType() const;+};++struct MapTypeInfo : Base<OrtMapTypeInfo> {+ explicit MapTypeInfo(std::nullptr_t) {}+ explicit MapTypeInfo(OrtMapTypeInfo* p) : Base<OrtMapTypeInfo>{p} {}++ ONNXTensorElementDataType GetMapKeyType() const;+ TypeInfo GetMapValueType() const;+};++struct TypeInfo : Base<OrtTypeInfo> {+ explicit TypeInfo(std::nullptr_t) {}+ explicit TypeInfo(OrtTypeInfo* p) : Base<OrtTypeInfo>{p} {}++ Unowned<TensorTypeAndShapeInfo> GetTensorTypeAndShapeInfo() const;+ Unowned<SequenceTypeInfo> GetSequenceTypeInfo() const;+ Unowned<MapTypeInfo> GetMapTypeInfo() const;++ ONNXType GetONNXType() const;+};++struct Value : Base<OrtValue> {+ template <typename T>+ static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len);+ static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,+ ONNXTensorElementDataType type);+ template <typename T>+ static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);+ static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type);++ static Value CreateMap(Value& keys, Value& values);+ static Value CreateSequence(std::vector<Value>& values);++ template <typename T>+ static Value CreateOpaque(const char* domain, const char* type_name, const T&);++ template <typename T>+ void GetOpaqueData(const char* domain, const char* type_name, T&) const;++ explicit Value(std::nullptr_t) {}+ explicit Value(OrtValue* p) : Base<OrtValue>{p} {}+ Value(Value&&) = default;+ Value& operator=(Value&&) = default;++ bool IsTensor() const;+ size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements+ Value GetValue(int index, OrtAllocator* allocator) const;++ size_t GetStringTensorDataLength() const;+ void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;++ template <typename T>+ T* GetTensorMutableData();++ template <typename T>+ const T* GetTensorData() const;++ template <typename T>+ T& At(const std::vector<int64_t>& location);++ TypeInfo GetTypeInfo() const;+ TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const;++ size_t GetStringTensorElementLength(size_t element_index) const;+ void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;++ void FillStringTensor(const char* const* s, size_t s_len);+ void FillStringTensorElement(const char* s, size_t index);+};++// Represents native memory allocation+struct MemoryAllocation {+ MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);+ ~MemoryAllocation();+ MemoryAllocation(const MemoryAllocation&) = delete;+ MemoryAllocation& operator=(const MemoryAllocation&) = delete;+ MemoryAllocation(MemoryAllocation&&);+ MemoryAllocation& operator=(MemoryAllocation&&);++ void* get() { return p_; }+ size_t size() const { return size_; }++ private:+ OrtAllocator* allocator_;+ void* p_;+ size_t size_;+};++struct AllocatorWithDefaultOptions {+ AllocatorWithDefaultOptions();++ operator OrtAllocator*() { return p_; }+ operator const OrtAllocator*() const { return p_; }++ void* Alloc(size_t size);+ // The return value will own the allocation+ MemoryAllocation GetAllocation(size_t size);+ void Free(void* p);++ const OrtMemoryInfo* GetInfo() const;++ private:+ OrtAllocator* p_{};+};++template <typename B>+struct BaseMemoryInfo : B {+ BaseMemoryInfo() = default;+ explicit BaseMemoryInfo(typename B::contained_type* p) : B(p) {}+ ~BaseMemoryInfo() = default;+ BaseMemoryInfo(BaseMemoryInfo&&) = default;+ BaseMemoryInfo& operator=(BaseMemoryInfo&&) = default;++ std::string GetAllocatorName() const;+ OrtAllocatorType GetAllocatorType() const;+ int GetDeviceId() const;+ OrtMemType GetMemoryType() const;+ template <typename U>+ bool operator==(const BaseMemoryInfo<U>& o) const;+};++struct UnownedMemoryInfo : BaseMemoryInfo<Base<const OrtMemoryInfo> > {+ explicit UnownedMemoryInfo(std::nullptr_t) {}+ explicit UnownedMemoryInfo(const OrtMemoryInfo* p) : BaseMemoryInfo(p) {}+};++struct MemoryInfo : BaseMemoryInfo<Base<OrtMemoryInfo> > {+ static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1);++ explicit MemoryInfo(std::nullptr_t) {}+ explicit MemoryInfo(OrtMemoryInfo* p) : BaseMemoryInfo(p) {}+ MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);+};++struct Allocator : public Base<OrtAllocator> {+ Allocator(const Session& session, const MemoryInfo&);++ void* Alloc(size_t size) const;+ // The return value will own the allocation+ MemoryAllocation GetAllocation(size_t size);+ void Free(void* p) const;+ UnownedMemoryInfo GetInfo() const;+};++struct IoBinding : public Base<OrtIoBinding> {+ private:+ std::vector<std::string> GetOutputNamesHelper(OrtAllocator*) const;+ std::vector<Value> GetOutputValuesHelper(OrtAllocator*) const;++ public:+ explicit IoBinding(Session& session);+ void BindInput(const char* name, const Value&);+ void BindOutput(const char* name, const Value&);+ void BindOutput(const char* name, const MemoryInfo&);+ std::vector<std::string> GetOutputNames() const;+ std::vector<std::string> GetOutputNames(Allocator&) const;+ std::vector<Value> GetOutputValues() const;+ std::vector<Value> GetOutputValues(Allocator&) const;+ void ClearBoundInputs();+ void ClearBoundOutputs();+};++/*! \struct Ort::ArenaCfg+ * \brief it is a structure that represents the configuration of an arena based allocator+ * \details Please see docs/C_API.md for details+ */+struct ArenaCfg : Base<OrtArenaCfg> {+ explicit ArenaCfg(std::nullptr_t) {}+ /**+ * \param max_mem - use 0 to allow ORT to choose the default+ * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested+ * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default+ * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default+ * \return an instance of ArenaCfg+ * See docs/C_API.md for details on what the following parameters mean and how to choose these values+ */+ ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);+};++//+// Custom OPs (only needed to implement custom OPs)+//++struct CustomOpApi {+ CustomOpApi(const OrtApi& api) : api_(api) {}++ template <typename T> // T is only implemented for float, int64_t, and string+ T KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name);++ OrtTensorTypeAndShapeInfo* GetTensorTypeAndShape(_In_ const OrtValue* value);+ size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info);+ ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo* info);+ size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info);+ void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length);+ void SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);++ template <typename T>+ T* GetTensorMutableData(_Inout_ OrtValue* value);+ template <typename T>+ const T* GetTensorData(_Inout_ const OrtValue* value);++ std::vector<int64_t> GetTensorShape(const OrtTensorTypeAndShapeInfo* info);+ void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input);+ size_t KernelContext_GetInputCount(const OrtKernelContext* context);+ const OrtValue* KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index);+ size_t KernelContext_GetOutputCount(const OrtKernelContext* context);+ OrtValue* KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count);++ void ThrowOnError(OrtStatus* result);++ private:+ const OrtApi& api_;+};++template <typename TOp, typename TKernel>+struct CustomOpBase : OrtCustomOp {+ CustomOpBase() {+ OrtCustomOp::version = ORT_API_VERSION;+ OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };+ OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };++ OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };++ OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };+ OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };++ OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };+ OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };++ OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { static_cast<TKernel*>(op_kernel)->Compute(context); };+ OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };+ }++ // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider+ const char* GetExecutionProviderType() const { return nullptr; }+};++} // namespace Ort++#include "onnxruntime_cxx_inline.h"diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.hnew file mode 100644index 0000000..a5ce821--- /dev/null+++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h@@ -0,0 +1,985 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead.+// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead.+//+// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter+// the main C++ file with implementation details.++namespace Ort {++inline void ThrowOnError(const OrtApi& ort, OrtStatus* status) {+ if (status) {+ std::string error_message = ort.GetErrorMessage(status);+ OrtErrorCode error_code = ort.GetErrorCode(status);+ ort.ReleaseStatus(status);+ ORT_CXX_API_THROW(std::move(error_message), error_code);+ }+}++inline void ThrowOnError(OrtStatus* status) {+ ThrowOnError(GetApi(), status);+}++// This template converts a C++ type into it's ONNXTensorElementDataType+template <typename T>+struct TypeToTensorType;+template <>+struct TypeToTensorType<float> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; };+template <>+struct TypeToTensorType<Float16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; };+template <>+struct TypeToTensorType<BFloat16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; };+template <>+struct TypeToTensorType<double> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; };+template <>+struct TypeToTensorType<int8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; };+template <>+struct TypeToTensorType<int16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; };+template <>+struct TypeToTensorType<int32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; };+template <>+struct TypeToTensorType<int64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; };+template <>+struct TypeToTensorType<uint8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; };+template <>+struct TypeToTensorType<uint16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; };+template <>+struct TypeToTensorType<uint32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; };+template <>+struct TypeToTensorType<uint64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; };+template <>+struct TypeToTensorType<bool> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; };++inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size)+ : allocator_(allocator), p_(p), size_(size) {+}++inline MemoryAllocation::~MemoryAllocation() {+ if (p_ != nullptr) {+ // We do not throw out of destructor+ auto ret = GetApi().AllocatorFree(allocator_, p_);+ static_cast<void>(ret);+ }+}++inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) : allocator_(nullptr), p_(nullptr), size_(0) {+ *this = std::move(o);+}++inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) {+ OrtAllocator* alloc = nullptr;+ void* p = nullptr;+ size_t sz = 0;++ // Swap out this+ std::swap(alloc, allocator_);+ std::swap(p, p_);+ std::swap(sz, size_);++ // Swap with incoming+ std::swap(allocator_, o.allocator_);+ std::swap(p_, o.p_);+ std::swap(size_, o.size_);++ // Destroy this instance if needed+ MemoryAllocation this_alloc(alloc, p, sz);+ return *this;+}++inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() {+ ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&p_));+}++inline void* AllocatorWithDefaultOptions::Alloc(size_t size) {+ void* out;+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));+ return out;+}++inline MemoryAllocation Ort::AllocatorWithDefaultOptions::GetAllocation(size_t size) {+ void* out;+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));+ MemoryAllocation result(p_, out, size);+ return result;+}++inline void AllocatorWithDefaultOptions::Free(void* p) {+ ThrowOnError(GetApi().AllocatorFree(p_, p));+}++inline const OrtMemoryInfo* AllocatorWithDefaultOptions::GetInfo() const {+ const OrtMemoryInfo* out;+ ThrowOnError(GetApi().AllocatorGetInfo(p_, &out));+ return out;+}++template <typename B>+inline std::string BaseMemoryInfo<B>::GetAllocatorName() const {+ const char* name = nullptr;+ ThrowOnError(GetApi().MemoryInfoGetName(*this, &name));+ return std::string(name);+}++template <typename B>+inline OrtAllocatorType BaseMemoryInfo<B>::GetAllocatorType() const {+ OrtAllocatorType type;+ ThrowOnError(GetApi().MemoryInfoGetType(*this, &type));+ return type;+}++template <typename B>+int BaseMemoryInfo<B>::GetDeviceId() const {+ int id = 0;+ ThrowOnError(GetApi().MemoryInfoGetId(*this, &id));+ return id;+}++template <typename B>+inline OrtMemType BaseMemoryInfo<B>::GetMemoryType() const {+ OrtMemType type;+ ThrowOnError(GetApi().MemoryInfoGetMemType(*this, &type));+ return type;+}++template <typename B>+template <typename U>+inline bool BaseMemoryInfo<B>::operator==(const BaseMemoryInfo<U>& o) const {+ int comp_result = 0;+ ThrowOnError(Ort::GetApi().CompareMemoryInfo(*this, o, &comp_result));+ return comp_result == 0;+}++inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) {+ OrtMemoryInfo* p;+ ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p));+ return MemoryInfo(p);+}++inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) {+ ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &p_));+}++inline Allocator::Allocator(const Session& sess, const MemoryInfo& mem_info) {+ ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &p_));+}++inline void* Allocator::Alloc(size_t size) const {+ void* out = nullptr;+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));+ return out;+}++inline MemoryAllocation Ort::Allocator::GetAllocation(size_t size) {+ void* out = nullptr;+ ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out));+ MemoryAllocation result(p_, out, size);+ return result;+}++inline void Allocator::Free(void* p) const {+ ThrowOnError(GetApi().AllocatorFree(p_, p));+}++inline UnownedMemoryInfo Allocator::GetInfo() const {+ const OrtMemoryInfo* out = nullptr;+ ThrowOnError(GetApi().AllocatorGetInfo(p_, &out));+ return UnownedMemoryInfo(out);+}++inline IoBinding::IoBinding(Session& session) {+ ThrowOnError(GetApi().CreateIoBinding(session, &p_));+}++inline void IoBinding::BindInput(const char* name, const Value& value) {+ ThrowOnError(GetApi().BindInput(p_, name, value));+}++inline void IoBinding::BindOutput(const char* name, const Value& value) {+ ThrowOnError(GetApi().BindOutput(p_, name, value));+}++inline void IoBinding::BindOutput(const char* name, const MemoryInfo& mem_info) {+ ThrowOnError(GetApi().BindOutputToDevice(p_, name, mem_info));+}++inline std::vector<std::string> IoBinding::GetOutputNamesHelper(OrtAllocator* allocator) const {+ std::vector<std::string> result;+ auto free_fn = [allocator](void* p) { if (p) allocator->Free(allocator, p); };+ using Ptr = std::unique_ptr<void, decltype(free_fn)>;++ char* buffer = nullptr;+ size_t* lengths = nullptr;+ size_t count = 0;+ ThrowOnError(GetApi().GetBoundOutputNames(p_, allocator, &buffer, &lengths, &count));++ if (count == 0) {+ return result;+ }++ Ptr buffer_g(buffer, free_fn);+ Ptr lengths_g(lengths, free_fn);++ result.reserve(count);+ for (size_t i = 0; i < count; ++i) {+ auto sz = *lengths;+ result.emplace_back(buffer, sz);+ buffer += sz;+ ++lengths;+ }+ return result;+}++inline std::vector<std::string> IoBinding::GetOutputNames() const {+ AllocatorWithDefaultOptions allocator;+ return GetOutputNamesHelper(allocator);+}++inline std::vector<std::string> IoBinding::GetOutputNames(Allocator& allocator) const {+ return GetOutputNamesHelper(allocator);+}++inline std::vector<Value> Ort::IoBinding::GetOutputValuesHelper(OrtAllocator* allocator) const {+ std::vector<Value> result;+ size_t owned = 0;+ size_t output_count = 0;+ // Lambda to release the buffer when no longer needed and+ // make sure that we destroy all instances on exception+ auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) {+ if (buffer) {+ while (owned < output_count) {+ auto* p = buffer + owned++;+ GetApi().ReleaseValue(*p);+ }+ allocator->Free(allocator, buffer);+ }+ };+ using Ptr = std::unique_ptr<OrtValue*, decltype(free_fn)>;++ OrtValue** output_buffer = nullptr;+ ThrowOnError(GetApi().GetBoundOutputValues(p_, allocator, &output_buffer, &output_count));+ if (output_count == 0) {+ return result;+ }++ Ptr buffer_g(output_buffer, free_fn);++ result.reserve(output_count);+ for (size_t i = 0; i < output_count; ++i) {+ result.emplace_back(output_buffer[i]);+ ++owned;+ }+ return result;+}++inline std::vector<Value> Ort::IoBinding::GetOutputValues(Allocator& allocator) const {+ return GetOutputValuesHelper(allocator);+}++inline std::vector<Value> Ort::IoBinding::GetOutputValues() const {+ AllocatorWithDefaultOptions allocator;+ return GetOutputValuesHelper(allocator);+}++inline void IoBinding::ClearBoundInputs() {+ GetApi().ClearBoundInputs(p_);+}++inline void IoBinding::ClearBoundOutputs() {+ GetApi().ClearBoundOutputs(p_);+}++inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) {+ ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_));+}++inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) {+ ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_));+ if (strcmp(logid, "onnxruntime-node") == 0) {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));+ } else {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));+ }+}++inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) {+ ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_));+ if (strcmp(logid, "onnxruntime-node") == 0) {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));+ } else {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));+ }+}++inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) {+ ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_));+ if (strcmp(logid, "onnxruntime-node") == 0) {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));+ } else {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));+ }+}++inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,+ OrtLoggingLevel logging_level, _In_ const char* logid) {+ ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_));+ if (strcmp(logid, "onnxruntime-node") == 0) {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS));+ } else {+ ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));+ }+}++inline Env& Env::EnableTelemetryEvents() {+ ThrowOnError(GetApi().EnableTelemetryEvents(p_));+ return *this;+}++inline Env& Env::DisableTelemetryEvents() {+ ThrowOnError(GetApi().DisableTelemetryEvents(p_));+ return *this;+}++inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) {+ ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg));+ return *this;+}++inline CustomOpDomain::CustomOpDomain(const char* domain) {+ ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_));+}++inline void CustomOpDomain::Add(OrtCustomOp* op) {+ ThrowOnError(GetApi().CustomOpDomain_Add(p_, op));+}++inline RunOptions::RunOptions() {+ ThrowOnError(GetApi().CreateRunOptions(&p_));+}++inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) {+ ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level));+ return *this;+}++inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) {+ ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level));+ return *this;+}++inline int RunOptions::GetRunLogVerbosityLevel() const {+ int out;+ ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out));+ return out;+}++inline RunOptions& RunOptions::SetRunTag(const char* run_tag) {+ ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag));+ return *this;+}++inline const char* RunOptions::GetRunTag() const {+ const char* out;+ ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out));+ return out;+}++inline RunOptions& RunOptions::SetTerminate() {+ ThrowOnError(GetApi().RunOptionsSetTerminate(p_));+ return *this;+}++inline RunOptions& RunOptions::UnsetTerminate() {+ ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_));+ return *this;+}++inline SessionOptions::SessionOptions() {+ ThrowOnError(GetApi().CreateSessionOptions(&p_));+}++inline SessionOptions SessionOptions::Clone() const {+ OrtSessionOptions* out;+ ThrowOnError(GetApi().CloneSessionOptions(p_, &out));+ return SessionOptions{out};+}++inline SessionOptions& SessionOptions::SetIntraOpNumThreads(int intra_op_num_threads) {+ ThrowOnError(GetApi().SetIntraOpNumThreads(p_, intra_op_num_threads));+ return *this;+}++inline SessionOptions& SessionOptions::SetInterOpNumThreads(int inter_op_num_threads) {+ ThrowOnError(GetApi().SetInterOpNumThreads(p_, inter_op_num_threads));+ return *this;+}++inline SessionOptions& SessionOptions::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) {+ ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(p_, graph_optimization_level));+ return *this;+}++inline SessionOptions& SessionOptions::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) {+ ThrowOnError(GetApi().SetOptimizedModelFilePath(p_, optimized_model_filepath));+ return *this;+}++inline SessionOptions& SessionOptions::EnableProfiling(const ORTCHAR_T* profile_file_prefix) {+ ThrowOnError(GetApi().EnableProfiling(p_, profile_file_prefix));+ return *this;+}++inline SessionOptions& SessionOptions::DisableProfiling() {+ ThrowOnError(GetApi().DisableProfiling(p_));+ return *this;+}++inline SessionOptions& SessionOptions::EnableMemPattern() {+ ThrowOnError(GetApi().EnableMemPattern(p_));+ return *this;+}++inline SessionOptions& SessionOptions::DisableMemPattern() {+ ThrowOnError(GetApi().DisableMemPattern(p_));+ return *this;+}++inline SessionOptions& SessionOptions::EnableCpuMemArena() {+ ThrowOnError(GetApi().EnableCpuMemArena(p_));+ return *this;+}++inline SessionOptions& SessionOptions::DisableCpuMemArena() {+ ThrowOnError(GetApi().DisableCpuMemArena(p_));+ return *this;+}++inline SessionOptions& SessionOptions::SetExecutionMode(ExecutionMode execution_mode) {+ ThrowOnError(GetApi().SetSessionExecutionMode(p_, execution_mode));+ return *this;+}++inline SessionOptions& SessionOptions::SetLogId(const char* logid) {+ ThrowOnError(GetApi().SetSessionLogId(p_, logid));+ return *this;+}++inline SessionOptions& SessionOptions::SetLogSeverityLevel(int level) {+ ThrowOnError(GetApi().SetSessionLogSeverityLevel(p_, level));+ return *this;+}++inline SessionOptions& SessionOptions::Add(OrtCustomOpDomain* custom_op_domain) {+ ThrowOnError(GetApi().AddCustomOpDomain(p_, custom_op_domain));+ return *this;+}++inline SessionOptions& SessionOptions::AddConfigEntry(const char* config_key, const char* config_value) {+ ThrowOnError(GetApi().AddSessionConfigEntry(p_, config_key, config_value));+ return *this;+}++inline SessionOptions& SessionOptions::AddInitializer(const char* name, const OrtValue* ort_val) {+ ThrowOnError(GetApi().AddInitializer(p_, name, ort_val));+ return *this;+}++inline SessionOptions& SessionOptions::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) {+ ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(p_, &provider_options));+ return *this;+}++inline SessionOptions& SessionOptions::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) {+ ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(p_, &provider_options));+ return *this;+}++inline Session::Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) {+ ThrowOnError(GetApi().CreateSession(env, model_path, options, &p_));+}++inline Session::Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) {+ ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &p_));+}++inline std::vector<Value> Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,+ const char* const* output_names, size_t output_names_count) {+ std::vector<Ort::Value> output_values;+ for (size_t i = 0; i < output_names_count; i++)+ output_values.emplace_back(nullptr);+ Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_names_count);+ return output_values;+}++inline void Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,+ const char* const* output_names, Value* output_values, size_t output_count) {+ static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");+ auto ort_input_values = reinterpret_cast<const OrtValue**>(const_cast<Value*>(input_values));+ auto ort_output_values = reinterpret_cast<OrtValue**>(output_values);+ ThrowOnError(GetApi().Run(p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values));+}++inline void Session::Run(const RunOptions& run_options, const IoBinding& io_binding) {+ ThrowOnError(GetApi().RunWithBinding(p_, run_options, io_binding));+}++inline size_t Session::GetInputCount() const {+ size_t out;+ ThrowOnError(GetApi().SessionGetInputCount(p_, &out));+ return out;+}++inline size_t Session::GetOutputCount() const {+ size_t out;+ ThrowOnError(GetApi().SessionGetOutputCount(p_, &out));+ return out;+}++inline size_t Session::GetOverridableInitializerCount() const {+ size_t out;+ ThrowOnError(GetApi().SessionGetOverridableInitializerCount(p_, &out));+ return out;+}++inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out));+ return out;+}++inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out));+ return out;+}++inline char* Session::GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out));+ return out;+}++inline char* Session::EndProfiling(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out));+ return out;+}++inline uint64_t Session::GetProfilingStartTimeNs() const {+ uint64_t out;+ ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(p_, &out));+ return out;+}++inline ModelMetadata Session::GetModelMetadata() const {+ OrtModelMetadata* out;+ ThrowOnError(GetApi().SessionGetModelMetadata(p_, &out));+ return ModelMetadata{out};+}++inline char* ModelMetadata::GetProducerName(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out));+ return out;+}++inline char* ModelMetadata::GetGraphName(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out));+ return out;+}++inline char* ModelMetadata::GetDomain(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out));+ return out;+}++inline char* ModelMetadata::GetDescription(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out));+ return out;+}++inline char* ModelMetadata::GetGraphDescription(OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out));+ return out;+}++inline char* ModelMetadata::LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const {+ char* out;+ ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out));+ return out;+}++inline char** ModelMetadata::GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const {+ char** out;+ ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys));+ return out;+}++inline int64_t ModelMetadata::GetVersion() const {+ int64_t out;+ ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out));+ return out;+}++inline TypeInfo Session::GetInputTypeInfo(size_t index) const {+ OrtTypeInfo* out;+ ThrowOnError(GetApi().SessionGetInputTypeInfo(p_, index, &out));+ return TypeInfo{out};+}++inline TypeInfo Session::GetOutputTypeInfo(size_t index) const {+ OrtTypeInfo* out;+ ThrowOnError(GetApi().SessionGetOutputTypeInfo(p_, index, &out));+ return TypeInfo{out};+}++inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const {+ OrtTypeInfo* out;+ ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(p_, index, &out));+ return TypeInfo{out};+}++inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const {+ ONNXTensorElementDataType out;+ ThrowOnError(GetApi().GetTensorElementType(p_, &out));+ return out;+}++inline size_t TensorTypeAndShapeInfo::GetElementCount() const {+ size_t out;+ ThrowOnError(GetApi().GetTensorShapeElementCount(p_, &out));+ return static_cast<size_t>(out);+}++inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const {+ size_t out;+ ThrowOnError(GetApi().GetDimensionsCount(p_, &out));+ return out;+}++inline void TensorTypeAndShapeInfo::GetDimensions(int64_t* values, size_t values_count) const {+ ThrowOnError(GetApi().GetDimensions(p_, values, values_count));+}++inline void TensorTypeAndShapeInfo::GetSymbolicDimensions(const char** values, size_t values_count) const {+ ThrowOnError(GetApi().GetSymbolicDimensions(p_, values, values_count));+}++inline std::vector<int64_t> TensorTypeAndShapeInfo::GetShape() const {+ std::vector<int64_t> out(GetDimensionsCount(), 0);+ GetDimensions(out.data(), out.size());+ return out;+}++inline Unowned<TensorTypeAndShapeInfo> TypeInfo::GetTensorTypeAndShapeInfo() const {+ const OrtTensorTypeAndShapeInfo* out;+ ThrowOnError(GetApi().CastTypeInfoToTensorInfo(p_, &out));+ return Unowned<TensorTypeAndShapeInfo>(const_cast<OrtTensorTypeAndShapeInfo*>(out));+}++inline Unowned<SequenceTypeInfo> TypeInfo::GetSequenceTypeInfo() const {+ const OrtSequenceTypeInfo* out;+ ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(p_, &out));+ return Unowned<SequenceTypeInfo>{const_cast<OrtSequenceTypeInfo*>(out)};+}++inline TypeInfo SequenceTypeInfo::GetSequenceElementType() const {+ OrtTypeInfo* output;+ ThrowOnError(GetApi().GetSequenceElementType(p_, &output));+ return TypeInfo{output};+}++inline Unowned<MapTypeInfo> TypeInfo::GetMapTypeInfo() const {+ const OrtMapTypeInfo* out;+ ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(p_, &out));+ return Unowned<MapTypeInfo>{const_cast<OrtMapTypeInfo*>(out)};+}++inline ONNXTensorElementDataType MapTypeInfo::GetMapKeyType() const {+ ONNXTensorElementDataType out;+ ThrowOnError(GetApi().GetMapKeyType(p_, &out));+ return out;+}++inline TypeInfo MapTypeInfo::GetMapValueType() const {+ OrtTypeInfo* output;+ ThrowOnError(GetApi().GetMapValueType(p_, &output));+ return TypeInfo{output};+}++inline ONNXType TypeInfo::GetONNXType() const {+ ONNXType out;+ ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(p_, &out));+ return out;+}++template <typename T>+inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) {+ return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType<T>::type);+}++inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,+ ONNXTensorElementDataType type) {+ OrtValue* out;+ ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out));+ return Value{out};+}++template <typename T>+inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) {+ return CreateTensor(allocator, shape, shape_len, TypeToTensorType<T>::type);+}++inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) {+ OrtValue* out;+ ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out));+ return Value{out};+}++inline Value Value::CreateMap(Value& keys, Value& values) {+ OrtValue* out;+ OrtValue* inputs[2] = {keys, values};+ ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out));+ return Value{out};+}++inline Value Value::CreateSequence(std::vector<Value>& values) {+ OrtValue* out;+ std::vector<OrtValue*> values_ort{values.data(), values.data() + values.size()};+ ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out));+ return Value{out};+}++template <typename T>+inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) {+ OrtValue* out;+ ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out));+ return Value{out};+}++template <typename T>+inline void Value::GetOpaqueData(const char* domain, const char* type_name, T& out) const {+ ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, p_, &out, sizeof(T)));+}++inline bool Value::IsTensor() const {+ int out;+ ThrowOnError(GetApi().IsTensor(p_, &out));+ return out != 0;+}++inline size_t Value::GetCount() const {+ size_t out;+ ThrowOnError(GetApi().GetValueCount(p_, &out));+ return out;+}++inline Value Value::GetValue(int index, OrtAllocator* allocator) const {+ OrtValue* out;+ ThrowOnError(GetApi().GetValue(p_, index, allocator, &out));+ return Value{out};+}++inline size_t Value::GetStringTensorDataLength() const {+ size_t out;+ ThrowOnError(GetApi().GetStringTensorDataLength(p_, &out));+ return out;+}++inline size_t Value::GetStringTensorElementLength(size_t element_index) const {+ size_t out;+ ThrowOnError(GetApi().GetStringTensorElementLength(p_, element_index, &out));+ return out;+}++inline void Value::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const {+ ThrowOnError(GetApi().GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count));+}++inline void Value::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const {+ ThrowOnError(GetApi().GetStringTensorElement(p_, buffer_length, element_index, buffer));+}++inline void Value::FillStringTensor(const char* const* s, size_t s_len) {+ ThrowOnError(GetApi().FillStringTensor(p_, s, s_len));+}++inline void Value::FillStringTensorElement(const char* s, size_t index) {+ ThrowOnError(GetApi().FillStringTensorElement(p_, s, index));+}++template <typename T>+T* Value::GetTensorMutableData() {+ T* out;+ ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out));+ return out;+}++template <typename T>+const T* Value::GetTensorData() const {+ T* out;+ ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out));+ return out;+}++template <typename T>+inline T& Value::At(const std::vector<int64_t>& location) {+ static_assert(!std::is_same<T, std::string>::value, "this api does not support std::string");+ T* out;+ ThrowOnError(GetApi().TensorAt(p_, location.data(), location.size(), (void**)&out));+ return *out;+}++inline TypeInfo Value::GetTypeInfo() const {+ OrtTypeInfo* output;+ ThrowOnError(GetApi().GetTypeInfo(p_, &output));+ return TypeInfo{output};+}++inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const {+ OrtTensorTypeAndShapeInfo* output;+ ThrowOnError(GetApi().GetTensorTypeAndShape(p_, &output));+ return TensorTypeAndShapeInfo{output};+}++//+// Custom OP API Inlines+//+inline void CustomOpApi::ThrowOnError(OrtStatus* status) {+ Ort::ThrowOnError(api_, status);+}++template <>+inline float CustomOpApi::KernelInfoGetAttribute<float>(_In_ const OrtKernelInfo* info, _In_ const char* name) {+ float out;+ ThrowOnError(api_.KernelInfoGetAttribute_float(info, name, &out));+ return out;+}++template <>+inline int64_t CustomOpApi::KernelInfoGetAttribute<int64_t>(_In_ const OrtKernelInfo* info, _In_ const char* name) {+ int64_t out;+ ThrowOnError(api_.KernelInfoGetAttribute_int64(info, name, &out));+ return out;+}++template <>+inline std::string CustomOpApi::KernelInfoGetAttribute<std::string>(_In_ const OrtKernelInfo* info, _In_ const char* name) {+ size_t size = 0;+ std::string out;+ OrtStatus* status = api_.KernelInfoGetAttribute_string(info, name, nullptr, &size);++ // The status should be ORT_INVALID_ARGUMENT because the size is insufficient to hold the string+ if (api_.GetErrorCode(status) == ORT_INVALID_ARGUMENT) {+ api_.ReleaseStatus(status);+ out.resize(size);+ ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size));+ out.resize(size - 1); // remove the terminating character '\0'+ } else {+ ThrowOnError(status);+ }+ return out;+}++inline OrtTensorTypeAndShapeInfo* CustomOpApi::GetTensorTypeAndShape(_In_ const OrtValue* value) {+ OrtTensorTypeAndShapeInfo* out;+ ThrowOnError(api_.GetTensorTypeAndShape(value, &out));+ return out;+}++inline size_t CustomOpApi::GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info) {+ size_t out;+ ThrowOnError(api_.GetTensorShapeElementCount(info, &out));+ return out;+}++inline ONNXTensorElementDataType CustomOpApi::GetTensorElementType(const OrtTensorTypeAndShapeInfo* info) {+ ONNXTensorElementDataType out;+ ThrowOnError(api_.GetTensorElementType(info, &out));+ return out;+}++inline size_t CustomOpApi::GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info) {+ size_t out;+ ThrowOnError(api_.GetDimensionsCount(info, &out));+ return out;+}++inline void CustomOpApi::GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) {+ ThrowOnError(api_.GetDimensions(info, dim_values, dim_values_length));+}++inline void CustomOpApi::SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count) {+ ThrowOnError(api_.SetDimensions(info, dim_values, dim_count));+}++template <typename T>+inline T* CustomOpApi::GetTensorMutableData(_Inout_ OrtValue* value) {+ T* data;+ ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast<void**>(&data)));+ return data;+}++template <typename T>+inline const T* CustomOpApi::GetTensorData(_Inout_ const OrtValue* value) {+ return GetTensorMutableData<T>(const_cast<OrtValue*>(value));+}++inline std::vector<int64_t> CustomOpApi::GetTensorShape(const OrtTensorTypeAndShapeInfo* info) {+ std::vector<int64_t> output(GetDimensionsCount(info));+ GetDimensions(info, output.data(), output.size());+ return output;+}++inline void CustomOpApi::ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input) {+ api_.ReleaseTensorTypeAndShapeInfo(input);+}++inline size_t CustomOpApi::KernelContext_GetInputCount(const OrtKernelContext* context) {+ size_t out;+ ThrowOnError(api_.KernelContext_GetInputCount(context, &out));+ return out;+}++inline const OrtValue* CustomOpApi::KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index) {+ const OrtValue* out;+ ThrowOnError(api_.KernelContext_GetInput(context, index, &out));+ return out;+}++inline size_t CustomOpApi::KernelContext_GetOutputCount(const OrtKernelContext* context) {+ size_t out;+ ThrowOnError(api_.KernelContext_GetOutputCount(context, &out));+ return out;+}++inline OrtValue* CustomOpApi::KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index,+ _In_ const int64_t* dim_values, size_t dim_count) {+ OrtValue* out;+ ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out));+ return out;+}++inline SessionOptions& SessionOptions::DisablePerSessionThreads() {+ ThrowOnError(GetApi().DisablePerSessionThreads(p_));+ return *this;+}++inline std::vector<std::string> GetAvailableProviders() {+ int len;+ char** providers;+ const OrtApi& api = GetApi();+ ThrowOnError(api.GetAvailableProviders(&providers, &len));+ std::vector<std::string> available_providers(providers, providers + len);+ ThrowOnError(api.ReleaseAvailableProviders(providers, len));+ return available_providers;+}++SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val);++} // namespace Ortdiff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.hnew file mode 100644index 0000000..ff247f2--- /dev/null+++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h@@ -0,0 +1,42 @@+// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License.++#pragma once++/*+ * This file defines SessionOptions Config Keys and format of the Config Values.+ *+ * The Naming Convention for a SessionOptions Config Key,+ * "[Area][.[SubArea1].[SubArea2]...].[Keyname]"+ * Such as "ep.cuda.use_arena"+ * The Config Key cannot be empty+ * The maximum length of the Config Key is 128+ *+ * The string format of a SessionOptions Config Value is defined individually for each Config.+ * The maximum length of the Config Value is 1024+ */++// Key for disable PrePacking,+// If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value)+static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking";++// A value of "1" means allocators registered in the env will be used. "0" means the allocators created in the session+// will be used. Use this to override the usage of env allocators on a per session level.+static const char* const kOrtSessionOptionsConfigUseEnvAllocators = "session.use_env_allocators";++// Set to 'ORT' (case sensitive) to load an ORT format model.+// If unset, model type will default to ONNX unless inferred from filename ('.ort' == ORT format) or bytes to be ORT+static const char* const kOrtSessionOptionsConfigLoadModelFormat = "session.load_model_format";++// Set to 'ORT' (case sensitive) to save optimized model in ORT format when SessionOptions.optimized_model_path is set.+// If unset, format will default to ONNX unless optimized_model_filepath ends in '.ort'.+static const char* const kOrtSessionOptionsConfigSaveModelFormat = "session.save_model_format";++// If a value is "1", flush-to-zero and denormal-as-zero are applied. The default is "0".+// When multiple sessions are created, a main thread doesn't override changes from succeeding session options,+// but threads in session thread pools follow option changes.+// When ORT runs with OpenMP, the same rule is applied, i.e. the first session option to flush-to-zero and+// denormal-as-zero is only applied to global OpenMP thread pool, which doesn't support per-session thread pool.+// Note that an alternative way not using this option at runtime is to train and export a model without denormals+// and that's recommended because turning this option on may hurt model accuracy.+static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.set_denormal_as_zero";