Core: Initial implementation of device memory mapping
Fernando Sahmkow
Liam
parent
3092855d5a
commit
2f0418c101
@ -0,0 +1,97 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
|
||||
#include "common/common_types.h"
|
||||
#include "common/virtual_buffer.h"
|
||||
|
||||
namespace Core {
|
||||
|
||||
class DeviceMemory;
|
||||
|
||||
namespace Memory {
|
||||
class Memory;
|
||||
}
|
||||
|
||||
template <typename DTraits>
|
||||
struct DeviceMemoryManagerAllocator;
|
||||
|
||||
template <typename Traits>
|
||||
class DeviceMemoryManager {
|
||||
using DeviceInterface = typename Traits::DeviceInterface;
|
||||
|
||||
public:
|
||||
DeviceMemoryManager(const DeviceMemory& device_memory);
|
||||
~DeviceMemoryManager();
|
||||
|
||||
void BindInterface(DeviceInterface* interface);
|
||||
|
||||
DAddr Allocate(size_t size);
|
||||
void AllocateFixed(DAddr start, size_t size);
|
||||
DAddr AllocatePinned(size_t size);
|
||||
void Free(DAddr start, size_t size);
|
||||
|
||||
void Map(DAddr address, VAddr virtual_address, size_t size, size_t p_id);
|
||||
void Unmap(DAddr address, size_t size);
|
||||
|
||||
// Write / Read
|
||||
template <typename T>
|
||||
T* GetPointer(DAddr address);
|
||||
|
||||
template <typename T>
|
||||
const T* GetPointer(DAddr address) const;
|
||||
|
||||
template <typename T>
|
||||
void Write(DAddr address, T value);
|
||||
|
||||
template <typename T>
|
||||
T Read(DAddr address) const;
|
||||
|
||||
void ReadBlock(DAddr address, void* dest_pointer, size_t size);
|
||||
void WriteBlock(DAddr address, void* src_pointer, size_t size);
|
||||
|
||||
size_t RegisterProcess(Memory::Memory* memory);
|
||||
void UnregisterProcess(size_t id);
|
||||
|
||||
private:
|
||||
static constexpr bool supports_pinning = Traits::supports_pinning;
|
||||
static constexpr size_t device_virtual_bits = Traits::device_virtual_bits;
|
||||
static constexpr size_t device_as_size = 1ULL << device_virtual_bits;
|
||||
static constexpr size_t physical_max_bits = 33;
|
||||
static constexpr size_t page_bits = 12;
|
||||
static constexpr u32 physical_address_base = 1U << page_bits;
|
||||
|
||||
template <typename T>
|
||||
T* GetPointerFromRaw(PAddr addr) {
|
||||
return reinterpret_cast<T*>(physical_base + addr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T* GetPointerFromRaw(PAddr addr) const {
|
||||
return reinterpret_cast<T*>(physical_base + addr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
PAddr GetRawPhysicalAddr(const T* ptr) const {
|
||||
return static_cast<PAddr>(reinterpret_cast<uintptr_t>(ptr) - physical_base);
|
||||
}
|
||||
|
||||
void WalkBlock(const DAddr addr, const std::size_t size, auto on_unmapped, auto on_memory,
|
||||
auto increment);
|
||||
|
||||
std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl;
|
||||
|
||||
const uintptr_t physical_base;
|
||||
DeviceInterface* interface;
|
||||
Common::VirtualBuffer<u32> compressed_physical_ptr;
|
||||
Common::VirtualBuffer<u32> compressed_device_addr;
|
||||
|
||||
std::deque<size_t> id_pool;
|
||||
std::deque<Memory::Memory*> registered_processes;
|
||||
};
|
||||
|
||||
} // namespace Core
|
||||
@ -0,0 +1,304 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
|
||||
#include "common/address_space.h"
|
||||
#include "common/address_space.inc"
|
||||
#include "common/alignment.h"
|
||||
#include "common/scope_exit.h"
|
||||
#include "core/device_memory.h"
|
||||
#include "core/device_memory_manager.h"
|
||||
#include "core/memory.h"
|
||||
|
||||
namespace Core {
|
||||
|
||||
struct EmptyAllocator {
|
||||
EmptyAllocator([[maybe_unused]] DAddr address) {}
|
||||
};
|
||||
|
||||
template <typename DTraits>
|
||||
struct DeviceMemoryManagerAllocator {
|
||||
static constexpr bool supports_pinning = DTraits::supports_pinning;
|
||||
static constexpr size_t device_virtual_bits = DTraits::device_virtual_bits;
|
||||
static constexpr size_t pin_bits = 32;
|
||||
static constexpr DAddr first_address = 1ULL << Memory::YUZU_PAGEBITS;
|
||||
static constexpr DAddr max_pin_area = supports_pinning ? 1ULL << pin_bits : first_address;
|
||||
static constexpr DAddr max_device_area = 1ULL << device_virtual_bits;
|
||||
|
||||
DeviceMemoryManagerAllocator()
|
||||
: pin_allocator(first_address),
|
||||
main_allocator(supports_pinning ? 1ULL << pin_bits : first_address) {}
|
||||
|
||||
std::conditional_t<supports_pinning, Common::FlatAllocator<DAddr, 0, pin_bits>, EmptyAllocator>
|
||||
pin_allocator;
|
||||
Common::FlatAllocator<DAddr, 0, device_virtual_bits> main_allocator;
|
||||
|
||||
/// Returns true when vaddr -> vaddr+size is fully contained in the buffer
|
||||
template <bool pin_area>
|
||||
[[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept {
|
||||
if constexpr (pin_area) {
|
||||
return addr >= 0 && addr + size <= max_pin_area;
|
||||
} else {
|
||||
return addr >= max_pin_area && addr + size <= max_device_area;
|
||||
}
|
||||
}
|
||||
|
||||
DAddr Allocate(size_t size) {
|
||||
return main_allocator.Allocate(size);
|
||||
}
|
||||
|
||||
DAddr AllocatePinned(size_t size) {
|
||||
return pin_allocator.Allocate(size);
|
||||
}
|
||||
|
||||
void DoInRange(DAddr address, size_t size, auto pin_func, auto main_func) {
|
||||
if (IsInBounds<true>(address, size)) {
|
||||
pin_func(address, size);
|
||||
return;
|
||||
}
|
||||
if (IsInBounds<false>(address, size)) {
|
||||
main_func(address, size);
|
||||
return;
|
||||
}
|
||||
DAddr end_size = address + size - max_pin_area;
|
||||
DAddr end_size2 = max_pin_area - address;
|
||||
pin_func(address, end_size2);
|
||||
main_func(max_pin_area, end_size);
|
||||
}
|
||||
|
||||
void AllocateFixed(DAddr b_address, size_t b_size) {
|
||||
if constexpr (supports_pinning) {
|
||||
DoInRange(
|
||||
b_address, b_size,
|
||||
[this](DAddr address, size_t size) { pin_allocator.AllocateFixed(address, size); },
|
||||
[this](DAddr address, size_t size) {
|
||||
main_allocator.AllocateFixed(address, size);
|
||||
});
|
||||
} else {
|
||||
main_allocator.AllocateFixed(b_address, b_size);
|
||||
}
|
||||
}
|
||||
|
||||
void Free(DAddr b_address, size_t b_size) {
|
||||
if constexpr (supports_pinning) {
|
||||
DoInRange(
|
||||
b_address, b_size,
|
||||
[this](DAddr address, size_t size) { pin_allocator.Free(address, size); },
|
||||
[this](DAddr address, size_t size) { main_allocator.Free(address, size); });
|
||||
} else {
|
||||
main_allocator.Free(b_address, b_size);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Traits>
|
||||
DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
|
||||
: physical_base{reinterpret_cast<const uintptr_t>(device_memory_.buffer.BackingBasePointer())},
|
||||
interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS),
|
||||
compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) {
|
||||
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
DeviceMemoryManager<Traits>::~DeviceMemoryManager() = default;
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::BindInterface(DeviceInterface* interface_) {
|
||||
interface = interface_;
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
DAddr DeviceMemoryManager<Traits>::Allocate(size_t size) {
|
||||
return impl->Allocate(size);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::AllocateFixed(DAddr start, size_t size) {
|
||||
return impl->AllocateFixed(start, size);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
DAddr DeviceMemoryManager<Traits>::AllocatePinned(size_t size) {
|
||||
return impl->AllocatePinned(size);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::Free(DAddr start, size_t size) {
|
||||
impl->Free(start, size);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size_t size,
|
||||
size_t p_id) {
|
||||
Core::Memory::Memory* process_memory = registered_processes[p_id];
|
||||
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
|
||||
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
for (size_t i = 0; i < num_pages; i++) {
|
||||
auto* ptr = process_memory->GetPointer(
|
||||
Common::ProcessAddress(virtual_address + i * Memory::YUZU_PAGESIZE));
|
||||
if (ptr == nullptr) [[unlikely]] {
|
||||
compressed_physical_ptr[start_page_d + i] = 0;
|
||||
continue;
|
||||
}
|
||||
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
|
||||
compressed_physical_ptr[start_page_d + i] = phys_addr;
|
||||
compressed_device_addr[phys_addr - 1U] = static_cast<u32>(start_page_d + i);
|
||||
}
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
|
||||
size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
|
||||
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
for (size_t i = 0; i < num_pages; i++) {
|
||||
auto phys_addr = compressed_physical_ptr[start_page_d + i];
|
||||
compressed_physical_ptr[start_page_d + i] = 0;
|
||||
if (phys_addr != 0) {
|
||||
compressed_device_addr[phys_addr - 1] = 0;
|
||||
}
|
||||
}
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
template <typename T>
|
||||
T* DeviceMemoryManager<Traits>::GetPointer(DAddr address) {
|
||||
const size_t index = address >> Memory::YUZU_PAGEBITS;
|
||||
const size_t offset = address & Memory::YUZU_PAGEMASK;
|
||||
auto phys_addr = compressed_physical_ptr[index];
|
||||
if (phys_addr == 0) [[unlikely]] {
|
||||
return nullptr;
|
||||
}
|
||||
return GetPointerFromRaw<T>(
|
||||
static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset));
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
template <typename T>
|
||||
const T* DeviceMemoryManager<Traits>::GetPointer(DAddr address) const {
|
||||
const size_t index = address >> Memory::YUZU_PAGEBITS;
|
||||
const size_t offset = address & Memory::YUZU_PAGEMASK;
|
||||
auto phys_addr = compressed_physical_ptr[index];
|
||||
if (phys_addr == 0) [[unlikely]] {
|
||||
return nullptr;
|
||||
}
|
||||
return GetPointerFromRaw<T>(
|
||||
static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset));
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
template <typename T>
|
||||
void DeviceMemoryManager<Traits>::Write(DAddr address, T value) {
|
||||
T* ptr = GetPointer<T>(address);
|
||||
if (!ptr) [[unlikely]] {
|
||||
return;
|
||||
}
|
||||
std::memcpy(ptr, &value, sizeof(T));
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
template <typename T>
|
||||
T DeviceMemoryManager<Traits>::Read(DAddr address) const {
|
||||
const T* ptr = GetPointer<T>(address);
|
||||
T result{};
|
||||
if (!ptr) [[unlikely]] {
|
||||
return result;
|
||||
}
|
||||
std::memcpy(&result, ptr, sizeof(T));
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto on_unmapped,
|
||||
auto on_memory, auto increment) {
|
||||
std::size_t remaining_size = size;
|
||||
std::size_t page_index = addr >> Memory::YUZU_PAGEBITS;
|
||||
std::size_t page_offset = addr & Memory::YUZU_PAGEMASK;
|
||||
|
||||
while (remaining_size) {
|
||||
const std::size_t copy_amount =
|
||||
std::min(static_cast<std::size_t>(Memory::YUZU_PAGESIZE) - page_offset, remaining_size);
|
||||
const auto current_vaddr =
|
||||
static_cast<u64>((page_index << Memory::YUZU_PAGEBITS) + page_offset);
|
||||
SCOPE_EXIT({
|
||||
page_index++;
|
||||
page_offset = 0;
|
||||
increment(copy_amount);
|
||||
remaining_size -= copy_amount;
|
||||
});
|
||||
|
||||
auto phys_addr = compressed_physical_ptr[page_index];
|
||||
if (phys_addr == 0) {
|
||||
on_unmapped(copy_amount, current_vaddr);
|
||||
continue;
|
||||
}
|
||||
auto* mem_ptr = GetPointerFromRaw<u8>(
|
||||
static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + page_offset));
|
||||
on_memory(copy_amount, mem_ptr);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, size_t size) {
|
||||
WalkBlock(
|
||||
address, size,
|
||||
[&](size_t copy_amount, DAddr current_vaddr) {
|
||||
LOG_ERROR(
|
||||
HW_Memory,
|
||||
"Unmapped Device ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
|
||||
current_vaddr, address, size);
|
||||
std::memset(dest_pointer, 0, copy_amount);
|
||||
},
|
||||
[&](size_t copy_amount, const u8* const src_ptr) {
|
||||
std::memcpy(dest_pointer, src_ptr, copy_amount);
|
||||
},
|
||||
[&](const std::size_t copy_amount) {
|
||||
dest_pointer = static_cast<u8*>(dest_pointer) + copy_amount;
|
||||
});
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::WriteBlock(DAddr address, void* src_pointer, size_t size) {
|
||||
WalkBlock(
|
||||
address, size,
|
||||
[&](size_t copy_amount, DAddr current_vaddr) {
|
||||
LOG_ERROR(
|
||||
HW_Memory,
|
||||
"Unmapped Device WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
|
||||
current_vaddr, address, size);
|
||||
},
|
||||
[&](size_t copy_amount, u8* const dst_ptr) {
|
||||
std::memcpy(dst_ptr, src_pointer, copy_amount);
|
||||
},
|
||||
[&](const std::size_t copy_amount) {
|
||||
src_pointer = static_cast<u8*>(src_pointer) + copy_amount;
|
||||
});
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
size_t DeviceMemoryManager<Traits>::RegisterProcess(Memory::Memory* memory_interface) {
|
||||
size_t new_id;
|
||||
if (!id_pool.empty()) {
|
||||
new_id = id_pool.front();
|
||||
id_pool.pop_front();
|
||||
registered_processes[new_id] = memory_interface;
|
||||
} else {
|
||||
registered_processes.emplace_back(memory_interface);
|
||||
new_id = registered_processes.size() - 1U;
|
||||
}
|
||||
return new_id;
|
||||
}
|
||||
|
||||
template <typename Traits>
|
||||
void DeviceMemoryManager<Traits>::UnregisterProcess(size_t id) {
|
||||
registered_processes[id] = nullptr;
|
||||
id_pool.push_front(id);
|
||||
}
|
||||
|
||||
} // namespace Core
|
||||
@ -0,0 +1,21 @@
|
||||
// SPDX-FileCopyrightText: 2023 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "core/device_memory_manager.inc"
|
||||
#include "video_core/host1x/gpu_device_memory_manager.h"
|
||||
#include "video_core/rasterizer_interface.h"
|
||||
|
||||
template struct Core::DeviceMemoryManagerAllocator<Tegra::MaxwellDeviceTraits>;
|
||||
template class Core::DeviceMemoryManager<Tegra::MaxwellDeviceTraits>;
|
||||
|
||||
template const u8* Tegra::MaxwellDeviceMemoryManager::GetPointer<u8>(DAddr addr) const;
|
||||
template u8* Tegra::MaxwellDeviceMemoryManager::GetPointer<u8>(DAddr addr);
|
||||
|
||||
template u8 Tegra::MaxwellDeviceMemoryManager::Read<u8>(DAddr addr) const;
|
||||
template u16 Tegra::MaxwellDeviceMemoryManager::Read<u16>(DAddr addr) const;
|
||||
template u32 Tegra::MaxwellDeviceMemoryManager::Read<u32>(DAddr addr) const;
|
||||
template u64 Tegra::MaxwellDeviceMemoryManager::Read<u64>(DAddr addr) const;
|
||||
template void Tegra::MaxwellDeviceMemoryManager::Write<u8>(DAddr addr, u8 data);
|
||||
template void Tegra::MaxwellDeviceMemoryManager::Write<u16>(DAddr addr, u16 data);
|
||||
template void Tegra::MaxwellDeviceMemoryManager::Write<u32>(DAddr addr, u32 data);
|
||||
template void Tegra::MaxwellDeviceMemoryManager::Write<u64>(DAddr addr, u64 data);
|
||||
@ -0,0 +1,20 @@
|
||||
// SPDX-FileCopyrightText: 2023 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "core/device_memory_manager.h"
|
||||
|
||||
namespace VideoCore {
|
||||
class RasterizerInterface;
|
||||
}
|
||||
|
||||
namespace Tegra {
|
||||
|
||||
struct MaxwellDeviceTraits {
|
||||
static constexpr bool supports_pinning = true;
|
||||
static constexpr size_t device_virtual_bits = 34;
|
||||
using DeviceInterface = typename VideoCore::RasterizerInterface;
|
||||
};
|
||||
|
||||
using MaxwellDeviceMemoryManager = Core::DeviceMemoryManager<MaxwellDeviceTraits>;
|
||||
|
||||
} // namespace Tegra
|
||||
Loading…
Reference in New Issue