Skip to content

VERY experimental Nintendo Switch 2 emulator

C++

[vulkan, vk_pipeline_cache] PSO Optimizations (#4294)

Lizzie and Samuel45b6ecff055c
diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp
index f9d9460c37..ddd52e9426 100644
--- a/src/video_core/renderer_vulkan/blit_image.cpp
+++ b/src/video_core/renderer_vulkan/blit_image.cpp
@@ -1281,7 +1281,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *blit_color_pipelines.back();
}
@@ -1313,7 +1313,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *blit_depth_stencil_pipelines.back();
}
@@ -1366,7 +1366,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *clear_color_pipelines.back();
}
@@ -1422,7 +1422,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *clear_stencil_pipelines.back();
}
@@ -1465,7 +1465,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *blit_msaa_color_pipelines.back();
}
@@ -1584,7 +1584,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPas
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *pipelines.back();
}
@@ -1697,7 +1697,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- }));
+ }, device.StaticPipelineCache()));
return *msaa_copy_pipelines.back();
}
@@ -1817,7 +1817,7 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass ren
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- });
+ }, device.StaticPipelineCache());
}
void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
@@ -1860,7 +1860,7 @@ void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass rende
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
- });
+ }, device.StaticPipelineCache());
}
} // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/present/util.cpp b/src/video_core/renderer_vulkan/present/util.cpp
index 11250687d6..2ea7870f01 100644
--- a/src/video_core/renderer_vulkan/present/util.cpp
+++ b/src/video_core/renderer_vulkan/present/util.cpp
@@ -521,7 +521,7 @@ static vk::Pipeline CreateWrappedPipelineImpl(
.subpass = 0,
.basePipelineHandle = 0,
.basePipelineIndex = 0,
- });
+ }, device.StaticPipelineCache());
}
vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass,
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
index 80c434dc8f..ab58560170 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
@@ -268,7 +268,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor
.layout = *layout,
.basePipelineHandle = {},
.basePipelineIndex = 0,
- });
+ }, device.StaticPipelineCache());
}
ComputePass::~ComputePass() = default;
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index 6e6120d1b1..642b9fa223 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -63,6 +63,8 @@ using VideoCommon::GenericEnvironment;
using VideoCommon::GraphicsEnvironment;
constexpr u32 CACHE_VERSION = 18;
+constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128;
+constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30;
constexpr std::array<char, 8> VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'};
template <typename Container>
@@ -710,6 +712,10 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
if (use_vulkan_pipeline_cache) {
SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
CACHE_VERSION);
+ size_t size = 0;
+ vulkan_pipeline_cache.Read(&size, nullptr);
+ last_cache_size.store(size, std::memory_order_relaxed);
+ last_flush = std::chrono::steady_clock::now();
}
if (state.statistics) {
@@ -717,6 +723,35 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
}
}
+void PipelineCache::QueueVulkanPipelineCacheFlush() {
+ if (!use_vulkan_pipeline_cache || vulkan_pipeline_cache_filename.empty()) {
+ return;
+ }
+ if (++pipelines_since_flush < VULKAN_CACHE_FLUSH_PIPELINES) {
+ return;
+ }
+ const auto now = std::chrono::steady_clock::now();
+ const auto megabytes = last_cache_size.load(std::memory_order_relaxed) / (1024 * 1024);
+ const std::chrono::seconds interval{
+ std::max<size_t>(VULKAN_CACHE_FLUSH_MIN_SECONDS, megabytes)};
+ if (last_flush.time_since_epoch().count() != 0 && now - last_flush < interval) {
+ return;
+ }
+ if (flush_in_flight.exchange(true, std::memory_order_acq_rel)) {
+ return;
+ }
+ pipelines_since_flush = 0;
+ last_flush = now;
+ serialization_thread.QueueWork([this] {
+ SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
+ CACHE_VERSION);
+ size_t size = 0;
+ vulkan_pipeline_cache.Read(&size, nullptr);
+ last_cache_size.store(size, std::memory_order_relaxed);
+ flush_in_flight.store(false, std::memory_order_release);
+ });
+}
+
GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)};
auto& pipeline{pair->second};
@@ -755,7 +790,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
std::span<Shader::Environment* const> envs, PipelineStatistics* statistics,
bool build_in_parallel) try {
auto hash = key.Hash();
- LOG_INFO(Render_Vulkan, "{:#016x}", hash);
+ LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
size_t env_index{0};
std::array<Shader::IR::Program, Maxwell::MaxShaderProgram> programs;
const bool uses_vertex_a{key.unique_hashes[0] != 0};
@@ -891,6 +926,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
}
SerializePipeline(key, env_ptrs, pipeline_cache_filename, CACHE_VERSION);
});
+ QueueVulkanPipelineCacheFlush();
return pipeline;
}
@@ -910,6 +946,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
SerializePipeline(key, std::array<const GenericEnvironment*, 1>{&env_},
pipeline_cache_filename, CACHE_VERSION);
});
+ QueueVulkanPipelineCacheFlush();
return pipeline;
}
@@ -922,7 +959,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
return nullptr;
}
- LOG_INFO(Render_Vulkan, "{:#016x}", hash);
+ LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()};
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
index ce89e981d2..9d66b663f5 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
@@ -7,6 +7,8 @@
#pragma once
#include <array>
+#include <atomic>
+#include <chrono>
#include <cstddef>
#include <filesystem>
#include <memory>
@@ -144,6 +146,8 @@ private:
vk::PipelineCache LoadVulkanPipelineCache(const std::filesystem::path& filename,
u32 expected_cache_version);
+ void QueueVulkanPipelineCacheFlush();
+
const Device& device;
Scheduler& scheduler;
DescriptorPool& descriptor_pool;
@@ -171,6 +175,10 @@ private:
std::filesystem::path vulkan_pipeline_cache_filename;
vk::PipelineCache vulkan_pipeline_cache;
+ size_t pipelines_since_flush{};
+ std::chrono::steady_clock::time_point last_flush{};
+ std::atomic<size_t> last_cache_size{};
+ std::atomic_bool flush_in_flight{};
Common::ThreadWorker workers;
Common::ThreadWorker serialization_thread;
diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index 9c0bc56a8b..b2cfd2c01d 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -7,6 +7,8 @@
#include <algorithm>
#include <bitset>
#include <chrono>
+#include <filesystem>
+#include <fstream>
#include <optional>
#include <thread>
#include <ankerl/unordered_dense.h>
@@ -16,6 +18,8 @@
#include <fmt/format.h>
#include "common/assert.h"
+#include "common/fs/fs.h"
+#include "common/fs/path_util.h"
#include "common/literals.h"
#include <ranges>
#include "common/settings.h"
@@ -393,6 +397,17 @@ std::vector<const char*> ExtensionListForVulkan(
return output;
}
+constexpr std::array<char, 8> STATIC_CACHE_MAGIC_NUMBER{'e', 'd', 'e', 'n', 's', 't', 'p', 'c'};
+constexpr u32 STATIC_CACHE_VERSION = 1;
+
+std::filesystem::path StaticPipelineCacheFilename() {
+ const auto shader_dir = Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir);
+ if (!Common::FS::CreateDir(shader_dir)) {
+ return {};
+ }
+ return shader_dir / "vulkan_static_pipelines.bin";
+}
+
} // Anonymous namespace
void Device::RemoveExtension(bool& extension, const std::string& extension_name) {
@@ -750,15 +765,100 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
vk::Check(vmaCreateAllocator(&allocator_info, &allocator));
+ owns_static_pipeline_cache = surface != VkSurfaceKHR{};
+ LoadStaticPipelineCache();
+
// Initialize GPU logging if enabled
InitializeGPULogging();
}
Device::~Device() {
+ SaveStaticPipelineCache();
ShutdownGPULogging();
vmaDestroyAllocator(allocator);
}
+void Device::LoadStaticPipelineCache() {
+ const auto create = [this](size_t size, const void* data) {
+ static_pipeline_cache = logical.CreatePipelineCache({
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .initialDataSize = size,
+ .pInitialData = data,
+ });
+ };
+ if (!owns_static_pipeline_cache) {
+ create(0, nullptr);
+ return;
+ }
+ const auto filename = StaticPipelineCacheFilename();
+ if (filename.empty()) {
+ create(0, nullptr);
+ return;
+ }
+ std::vector<char> data;
+ try {
+ std::ifstream file(filename, std::ios::binary | std::ios::ate);
+ if (!file.is_open()) {
+ create(0, nullptr);
+ return;
+ }
+ file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+ const size_t total = static_cast<size_t>(file.tellg());
+ file.seekg(0, std::ios::beg);
+ std::array<char, 8> magic{};
+ u32 version{};
+ if (total < magic.size() + sizeof(version)) {
+ create(0, nullptr);
+ return;
+ }
+ file.read(magic.data(), magic.size())
+ .read(reinterpret_cast<char*>(&version), sizeof(version));
+ if (magic != STATIC_CACHE_MAGIC_NUMBER || version != STATIC_CACHE_VERSION) {
+ create(0, nullptr);
+ return;
+ }
+ data.resize(total - magic.size() - sizeof(version));
+ file.read(data.data(), static_cast<std::streamsize>(data.size()));
+ } catch (const std::ios_base::failure& e) {
+ create(0, nullptr);
+ return;
+ }
+ create(data.size(), data.empty() ? nullptr : data.data());
+}
+
+void Device::SaveStaticPipelineCache() const {
+ if (!owns_static_pipeline_cache || !static_pipeline_cache) {
+ return;
+ }
+ const auto filename = StaticPipelineCacheFilename();
+ if (filename.empty()) {
+ return;
+ }
+ size_t size = 0;
+ std::vector<char> data;
+ static_pipeline_cache.Read(&size, nullptr);
+ if (size == 0) {
+ return;
+ }
+ data.resize(size);
+ static_pipeline_cache.Read(&size, data.data());
+ try {
+ std::ofstream file(filename, std::ios::binary | std::ios::trunc);
+ file.exceptions(std::ofstream::failbit);
+ if (!file.is_open()) {
+ return;
+ }
+ file.write(STATIC_CACHE_MAGIC_NUMBER.data(), STATIC_CACHE_MAGIC_NUMBER.size())
+ .write(reinterpret_cast<const char*>(&STATIC_CACHE_VERSION),
+ sizeof(STATIC_CACHE_VERSION))
+ .write(data.data(), static_cast<std::streamsize>(size));
+ } catch (const std::ios_base::failure& e) {
+ Common::FS::RemoveFile(filename);
+ }
+}
+
VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage,
FormatType format_type) const {
if (IsFormatSupported(wanted_format, wanted_usage, format_type)) {
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index 27e6636bdd..319a7b2ec3 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -273,6 +273,10 @@ public:
return physical;
}
+ VkPipelineCache StaticPipelineCache() const noexcept {
+ return *static_pipeline_cache;
+ }
+
/// Returns the main graphics queue.
vk::Queue GetGraphicsQueue() const {
return graphics_queue;
@@ -1127,6 +1131,9 @@ private:
/// Returns true if the device natively supports blitting depth stencil images.
bool TestDepthStencilBlits(VkFormat format) const;
+ void LoadStaticPipelineCache();
+ void SaveStaticPipelineCache() const;
+
private:
VkInstance instance; ///< Vulkan instance.
VmaAllocator allocator; ///< VMA allocator.
@@ -1135,6 +1142,8 @@ private:
vk::Device logical; ///< Logical device.
vk::Queue graphics_queue; ///< Main graphics queue.
vk::Queue present_queue; ///< Main present queue.
+ vk::PipelineCache static_pipeline_cache;
+ bool owns_static_pipeline_cache{};
u32 instance_version{}; ///< Vulkan instance version.
u32 graphics_family{}; ///< Main graphics queue family index.
u32 present_family{}; ///< Main present queue family index.