Merge pull request #3986 from ReinUsesLisp/shader-cache
shader_cache: Implement a generic runtime shader cachemaster
bunnei
GitHub
commit
5633887569
@ -1,7 +0,0 @@
|
|||||||
// Copyright 2018 yuzu Emulator Project
|
|
||||||
// Licensed under GPLv2 or any later version
|
|
||||||
// Refer to the license.txt file included.
|
|
||||||
|
|
||||||
#include "video_core/rasterizer_cache.h"
|
|
||||||
|
|
||||||
RasterizerCacheObject::~RasterizerCacheObject() = default;
|
|
||||||
@ -1,253 +0,0 @@
|
|||||||
// Copyright 2018 yuzu Emulator Project
|
|
||||||
// Licensed under GPLv2 or any later version
|
|
||||||
// Refer to the license.txt file included.
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <mutex>
|
|
||||||
#include <set>
|
|
||||||
#include <unordered_map>
|
|
||||||
|
|
||||||
#include <boost/icl/interval_map.hpp>
|
|
||||||
#include <boost/range/iterator_range_core.hpp>
|
|
||||||
|
|
||||||
#include "common/common_types.h"
|
|
||||||
#include "core/settings.h"
|
|
||||||
#include "video_core/gpu.h"
|
|
||||||
#include "video_core/rasterizer_interface.h"
|
|
||||||
|
|
||||||
class RasterizerCacheObject {
|
|
||||||
public:
|
|
||||||
explicit RasterizerCacheObject(const VAddr cpu_addr) : cpu_addr{cpu_addr} {}
|
|
||||||
|
|
||||||
virtual ~RasterizerCacheObject();
|
|
||||||
|
|
||||||
VAddr GetCpuAddr() const {
|
|
||||||
return cpu_addr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Gets the size of the shader in guest memory, required for cache management
|
|
||||||
virtual std::size_t GetSizeInBytes() const = 0;
|
|
||||||
|
|
||||||
/// Sets whether the cached object should be considered registered
|
|
||||||
void SetIsRegistered(bool registered) {
|
|
||||||
is_registered = registered;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns true if the cached object is registered
|
|
||||||
bool IsRegistered() const {
|
|
||||||
return is_registered;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns true if the cached object is dirty
|
|
||||||
bool IsDirty() const {
|
|
||||||
return is_dirty;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns ticks from when this cached object was last modified
|
|
||||||
u64 GetLastModifiedTicks() const {
|
|
||||||
return last_modified_ticks;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Marks an object as recently modified, used to specify whether it is clean or dirty
|
|
||||||
template <class T>
|
|
||||||
void MarkAsModified(bool dirty, T& cache) {
|
|
||||||
is_dirty = dirty;
|
|
||||||
last_modified_ticks = cache.GetModifiedTicks();
|
|
||||||
}
|
|
||||||
|
|
||||||
void SetMemoryMarked(bool is_memory_marked_) {
|
|
||||||
is_memory_marked = is_memory_marked_;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool IsMemoryMarked() const {
|
|
||||||
return is_memory_marked;
|
|
||||||
}
|
|
||||||
|
|
||||||
void SetSyncPending(bool is_sync_pending_) {
|
|
||||||
is_sync_pending = is_sync_pending_;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool IsSyncPending() const {
|
|
||||||
return is_sync_pending;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
bool is_registered{}; ///< Whether the object is currently registered with the cache
|
|
||||||
bool is_dirty{}; ///< Whether the object is dirty (out of sync with guest memory)
|
|
||||||
bool is_memory_marked{}; ///< Whether the object is marking rasterizer memory.
|
|
||||||
bool is_sync_pending{}; ///< Whether the object is pending deletion.
|
|
||||||
u64 last_modified_ticks{}; ///< When the object was last modified, used for in-order flushing
|
|
||||||
VAddr cpu_addr{}; ///< Cpu address memory, unique from emulated virtual address space
|
|
||||||
};
|
|
||||||
|
|
||||||
template <class T>
|
|
||||||
class RasterizerCache : NonCopyable {
|
|
||||||
friend class RasterizerCacheObject;
|
|
||||||
|
|
||||||
public:
|
|
||||||
explicit RasterizerCache(VideoCore::RasterizerInterface& rasterizer) : rasterizer{rasterizer} {}
|
|
||||||
|
|
||||||
/// Write any cached resources overlapping the specified region back to memory
|
|
||||||
void FlushRegion(VAddr addr, std::size_t size) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
const auto& objects{GetSortedObjectsFromRegion(addr, size)};
|
|
||||||
for (auto& object : objects) {
|
|
||||||
FlushObject(object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Mark the specified region as being invalidated
|
|
||||||
void InvalidateRegion(VAddr addr, u64 size) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
const auto& objects{GetSortedObjectsFromRegion(addr, size)};
|
|
||||||
for (auto& object : objects) {
|
|
||||||
if (!object->IsRegistered()) {
|
|
||||||
// Skip duplicates
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Unregister(object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnCPUWrite(VAddr addr, std::size_t size) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
for (const auto& object : GetSortedObjectsFromRegion(addr, size)) {
|
|
||||||
if (object->IsRegistered()) {
|
|
||||||
UnmarkMemory(object);
|
|
||||||
object->SetSyncPending(true);
|
|
||||||
marked_for_unregister.emplace_back(object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void SyncGuestHost() {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
for (const auto& object : marked_for_unregister) {
|
|
||||||
if (object->IsRegistered()) {
|
|
||||||
object->SetSyncPending(false);
|
|
||||||
Unregister(object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
marked_for_unregister.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Invalidates everything in the cache
|
|
||||||
void InvalidateAll() {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
while (interval_cache.begin() != interval_cache.end()) {
|
|
||||||
Unregister(*interval_cache.begin()->second.begin());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected:
|
|
||||||
/// Tries to get an object from the cache with the specified cache address
|
|
||||||
T TryGet(VAddr addr) const {
|
|
||||||
const auto iter = map_cache.find(addr);
|
|
||||||
if (iter != map_cache.end())
|
|
||||||
return iter->second;
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Register an object into the cache
|
|
||||||
virtual void Register(const T& object) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
object->SetIsRegistered(true);
|
|
||||||
interval_cache.add({GetInterval(object), ObjectSet{object}});
|
|
||||||
map_cache.insert({object->GetCpuAddr(), object});
|
|
||||||
rasterizer.UpdatePagesCachedCount(object->GetCpuAddr(), object->GetSizeInBytes(), 1);
|
|
||||||
object->SetMemoryMarked(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Unregisters an object from the cache
|
|
||||||
virtual void Unregister(const T& object) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
UnmarkMemory(object);
|
|
||||||
object->SetIsRegistered(false);
|
|
||||||
if (object->IsSyncPending()) {
|
|
||||||
marked_for_unregister.remove(object);
|
|
||||||
object->SetSyncPending(false);
|
|
||||||
}
|
|
||||||
const VAddr addr = object->GetCpuAddr();
|
|
||||||
interval_cache.subtract({GetInterval(object), ObjectSet{object}});
|
|
||||||
map_cache.erase(addr);
|
|
||||||
}
|
|
||||||
|
|
||||||
void UnmarkMemory(const T& object) {
|
|
||||||
if (!object->IsMemoryMarked()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
rasterizer.UpdatePagesCachedCount(object->GetCpuAddr(), object->GetSizeInBytes(), -1);
|
|
||||||
object->SetMemoryMarked(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns a ticks counter used for tracking when cached objects were last modified
|
|
||||||
u64 GetModifiedTicks() {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
return ++modified_ticks;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void FlushObjectInner(const T& object) = 0;
|
|
||||||
|
|
||||||
/// Flushes the specified object, updating appropriate cache state as needed
|
|
||||||
void FlushObject(const T& object) {
|
|
||||||
std::lock_guard lock{mutex};
|
|
||||||
|
|
||||||
if (!object->IsDirty()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
FlushObjectInner(object);
|
|
||||||
object->MarkAsModified(false, *this);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::recursive_mutex mutex;
|
|
||||||
|
|
||||||
private:
|
|
||||||
/// Returns a list of cached objects from the specified memory region, ordered by access time
|
|
||||||
std::vector<T> GetSortedObjectsFromRegion(VAddr addr, u64 size) {
|
|
||||||
if (size == 0) {
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<T> objects;
|
|
||||||
const ObjectInterval interval{addr, addr + size};
|
|
||||||
for (auto& pair : boost::make_iterator_range(interval_cache.equal_range(interval))) {
|
|
||||||
for (auto& cached_object : pair.second) {
|
|
||||||
if (!cached_object) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
objects.push_back(cached_object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::sort(objects.begin(), objects.end(), [](const T& a, const T& b) -> bool {
|
|
||||||
return a->GetLastModifiedTicks() < b->GetLastModifiedTicks();
|
|
||||||
});
|
|
||||||
|
|
||||||
return objects;
|
|
||||||
}
|
|
||||||
|
|
||||||
using ObjectSet = std::set<T>;
|
|
||||||
using ObjectCache = std::unordered_map<VAddr, T>;
|
|
||||||
using IntervalCache = boost::icl::interval_map<VAddr, ObjectSet>;
|
|
||||||
using ObjectInterval = typename IntervalCache::interval_type;
|
|
||||||
|
|
||||||
static auto GetInterval(const T& object) {
|
|
||||||
return ObjectInterval::right_open(object->GetCpuAddr(),
|
|
||||||
object->GetCpuAddr() + object->GetSizeInBytes());
|
|
||||||
}
|
|
||||||
|
|
||||||
ObjectCache map_cache;
|
|
||||||
IntervalCache interval_cache; ///< Cache of objects
|
|
||||||
u64 modified_ticks{}; ///< Counter of cache state ticks, used for in-order flushing
|
|
||||||
VideoCore::RasterizerInterface& rasterizer;
|
|
||||||
std::list<T> marked_for_unregister;
|
|
||||||
};
|
|
||||||
@ -0,0 +1,228 @@
|
|||||||
|
// Copyright 2020 yuzu Emulator Project
|
||||||
|
// Licensed under GPLv2 or any later version
|
||||||
|
// Refer to the license.txt file included.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <memory>
|
||||||
|
#include <mutex>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "common/assert.h"
|
||||||
|
#include "common/common_types.h"
|
||||||
|
#include "video_core/rasterizer_interface.h"
|
||||||
|
|
||||||
|
namespace VideoCommon {
|
||||||
|
|
||||||
|
template <class T>
|
||||||
|
class ShaderCache {
|
||||||
|
static constexpr u64 PAGE_SHIFT = 14;
|
||||||
|
|
||||||
|
struct Entry {
|
||||||
|
VAddr addr_start;
|
||||||
|
VAddr addr_end;
|
||||||
|
T* data;
|
||||||
|
|
||||||
|
bool is_memory_marked = true;
|
||||||
|
|
||||||
|
constexpr bool Overlaps(VAddr start, VAddr end) const noexcept {
|
||||||
|
return start < addr_end && addr_start < end;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
virtual ~ShaderCache() = default;
|
||||||
|
|
||||||
|
/// @brief Removes shaders inside a given region
|
||||||
|
/// @note Checks for ranges
|
||||||
|
/// @param addr Start address of the invalidation
|
||||||
|
/// @param size Number of bytes of the invalidation
|
||||||
|
void InvalidateRegion(VAddr addr, std::size_t size) {
|
||||||
|
std::scoped_lock lock{invalidation_mutex};
|
||||||
|
InvalidatePagesInRegion(addr, size);
|
||||||
|
RemovePendingShaders();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Unmarks a memory region as cached and marks it for removal
|
||||||
|
/// @param addr Start address of the CPU write operation
|
||||||
|
/// @param size Number of bytes of the CPU write operation
|
||||||
|
void OnCPUWrite(VAddr addr, std::size_t size) {
|
||||||
|
std::lock_guard lock{invalidation_mutex};
|
||||||
|
InvalidatePagesInRegion(addr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Flushes delayed removal operations
|
||||||
|
void SyncGuestHost() {
|
||||||
|
std::scoped_lock lock{invalidation_mutex};
|
||||||
|
RemovePendingShaders();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Tries to obtain a cached shader starting in a given address
|
||||||
|
/// @note Doesn't check for ranges, the given address has to be the start of the shader
|
||||||
|
/// @param addr Start address of the shader, this doesn't cache for region
|
||||||
|
/// @return Pointer to a valid shader, nullptr when nothing is found
|
||||||
|
T* TryGet(VAddr addr) const {
|
||||||
|
std::scoped_lock lock{lookup_mutex};
|
||||||
|
|
||||||
|
const auto it = lookup_cache.find(addr);
|
||||||
|
if (it == lookup_cache.end()) {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
return it->second->data;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
explicit ShaderCache(VideoCore::RasterizerInterface& rasterizer_) : rasterizer{rasterizer_} {}
|
||||||
|
|
||||||
|
/// @brief Register in the cache a given entry
|
||||||
|
/// @param data Shader to store in the cache
|
||||||
|
/// @param addr Start address of the shader that will be registered
|
||||||
|
/// @param size Size in bytes of the shader
|
||||||
|
void Register(std::unique_ptr<T> data, VAddr addr, std::size_t size) {
|
||||||
|
std::scoped_lock lock{invalidation_mutex, lookup_mutex};
|
||||||
|
|
||||||
|
const VAddr addr_end = addr + size;
|
||||||
|
Entry* const entry = NewEntry(addr, addr_end, data.get());
|
||||||
|
|
||||||
|
const u64 page_end = addr_end >> PAGE_SHIFT;
|
||||||
|
for (u64 page = addr >> PAGE_SHIFT; page <= page_end; ++page) {
|
||||||
|
invalidation_cache[page].push_back(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
storage.push_back(std::move(data));
|
||||||
|
|
||||||
|
rasterizer.UpdatePagesCachedCount(addr, size, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Called when a shader is going to be removed
|
||||||
|
/// @param shader Shader that will be removed
|
||||||
|
/// @pre invalidation_cache is locked
|
||||||
|
/// @pre lookup_mutex is locked
|
||||||
|
virtual void OnShaderRemoval([[maybe_unused]] T* shader) {}
|
||||||
|
|
||||||
|
private:
|
||||||
|
/// @brief Invalidate pages in a given region
|
||||||
|
/// @pre invalidation_mutex is locked
|
||||||
|
void InvalidatePagesInRegion(VAddr addr, std::size_t size) {
|
||||||
|
const VAddr addr_end = addr + size;
|
||||||
|
const u64 page_end = addr_end >> PAGE_SHIFT;
|
||||||
|
for (u64 page = addr >> PAGE_SHIFT; page <= page_end; ++page) {
|
||||||
|
const auto it = invalidation_cache.find(page);
|
||||||
|
if (it == invalidation_cache.end()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<Entry*>& entries = it->second;
|
||||||
|
InvalidatePageEntries(entries, addr, addr_end);
|
||||||
|
|
||||||
|
// If there's nothing else in this page, remove it to avoid overpopulating the hash map.
|
||||||
|
if (entries.empty()) {
|
||||||
|
invalidation_cache.erase(it);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Remove shaders marked for deletion
|
||||||
|
/// @pre invalidation_mutex is locked
|
||||||
|
void RemovePendingShaders() {
|
||||||
|
if (marked_for_removal.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::scoped_lock lock{lookup_mutex};
|
||||||
|
|
||||||
|
std::vector<T*> removed_shaders;
|
||||||
|
removed_shaders.reserve(marked_for_removal.size());
|
||||||
|
|
||||||
|
for (Entry* const entry : marked_for_removal) {
|
||||||
|
if (lookup_cache.erase(entry->addr_start) > 0) {
|
||||||
|
removed_shaders.push_back(entry->data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
marked_for_removal.clear();
|
||||||
|
|
||||||
|
if (!removed_shaders.empty()) {
|
||||||
|
RemoveShadersFromStorage(std::move(removed_shaders));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Invalidates entries in a given range for the passed page
|
||||||
|
/// @param entries Vector of entries in the page, it will be modified on overlaps
|
||||||
|
/// @param addr Start address of the invalidation
|
||||||
|
/// @param addr_end Non-inclusive end address of the invalidation
|
||||||
|
/// @pre invalidation_mutex is locked
|
||||||
|
void InvalidatePageEntries(std::vector<Entry*>& entries, VAddr addr, VAddr addr_end) {
|
||||||
|
auto it = entries.begin();
|
||||||
|
while (it != entries.end()) {
|
||||||
|
Entry* const entry = *it;
|
||||||
|
if (!entry->Overlaps(addr, addr_end)) {
|
||||||
|
++it;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
UnmarkMemory(entry);
|
||||||
|
marked_for_removal.push_back(entry);
|
||||||
|
|
||||||
|
it = entries.erase(it);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Unmarks an entry from the rasterizer cache
|
||||||
|
/// @param entry Entry to unmark from memory
|
||||||
|
void UnmarkMemory(Entry* entry) {
|
||||||
|
if (!entry->is_memory_marked) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
entry->is_memory_marked = false;
|
||||||
|
|
||||||
|
const VAddr addr = entry->addr_start;
|
||||||
|
const std::size_t size = entry->addr_end - addr;
|
||||||
|
rasterizer.UpdatePagesCachedCount(addr, size, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Removes a vector of shaders from a list
|
||||||
|
/// @param removed_shaders Shaders to be removed from the storage, it can contain duplicates
|
||||||
|
/// @pre invalidation_mutex is locked
|
||||||
|
/// @pre lookup_mutex is locked
|
||||||
|
void RemoveShadersFromStorage(std::vector<T*> removed_shaders) {
|
||||||
|
// Remove duplicates
|
||||||
|
std::sort(removed_shaders.begin(), removed_shaders.end());
|
||||||
|
removed_shaders.erase(std::unique(removed_shaders.begin(), removed_shaders.end()),
|
||||||
|
removed_shaders.end());
|
||||||
|
|
||||||
|
// Now that there are no duplicates, we can notify removals
|
||||||
|
for (T* const shader : removed_shaders) {
|
||||||
|
OnShaderRemoval(shader);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove them from the cache
|
||||||
|
const auto is_removed = [&removed_shaders](std::unique_ptr<T>& shader) {
|
||||||
|
return std::find(removed_shaders.begin(), removed_shaders.end(), shader.get()) !=
|
||||||
|
removed_shaders.end();
|
||||||
|
};
|
||||||
|
storage.erase(std::remove_if(storage.begin(), storage.end(), is_removed), storage.end());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// @brief Creates a new entry in the lookup cache and returns its pointer
|
||||||
|
/// @pre lookup_mutex is locked
|
||||||
|
Entry* NewEntry(VAddr addr, VAddr addr_end, T* data) {
|
||||||
|
auto entry = std::make_unique<Entry>(Entry{addr, addr_end, data});
|
||||||
|
Entry* const entry_pointer = entry.get();
|
||||||
|
|
||||||
|
lookup_cache.emplace(addr, std::move(entry));
|
||||||
|
return entry_pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
VideoCore::RasterizerInterface& rasterizer;
|
||||||
|
|
||||||
|
mutable std::mutex lookup_mutex;
|
||||||
|
std::mutex invalidation_mutex;
|
||||||
|
|
||||||
|
std::unordered_map<u64, std::unique_ptr<Entry>> lookup_cache;
|
||||||
|
std::unordered_map<u64, std::vector<Entry*>> invalidation_cache;
|
||||||
|
std::vector<std::unique_ptr<T>> storage;
|
||||||
|
std::vector<Entry*> marked_for_removal;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace VideoCommon
|
||||||
Loading…
Reference in New Issue