core/memory: Read and write page table atomically

Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.

Configure Dynarmic to mask out these bits on pointer reads.

While we are at it, remove some unused attributes carried over from
Citra.

Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:

 mov         rdi,rcx
 shr         rdx,0Ch
 mov         r8,qword ptr [rax+8]
 mov         rax,qword ptr [r8+rdx*8]
 mov         rdx,rax
-and         al,3
 and         rdx,0FFFFFFFFFFFFFFFCh
 je          Core::Memory::Memory::Impl::Read<unsigned char>
 mov         rax,qword ptr [vaddr]
 movzx       eax,byte ptr [rdx+rax]
master
ReinUsesLisp 2020-12-29 21:16:57 +07:00
parent 8d55c8c855
commit b3587102d1
7 changed files with 132 additions and 147 deletions

@ -10,16 +10,10 @@ PageTable::PageTable() = default;
PageTable::~PageTable() noexcept = default; PageTable::~PageTable() noexcept = default;
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, void PageTable::Resize(size_t address_space_width_in_bits, size_t page_size_in_bits) {
bool has_attribute) { const size_t num_page_table_entries{1ULL << (address_space_width_in_bits - page_size_in_bits)};
const std::size_t num_page_table_entries{1ULL
<< (address_space_width_in_bits - page_size_in_bits)};
pointers.resize(num_page_table_entries); pointers.resize(num_page_table_entries);
backing_addr.resize(num_page_table_entries); backing_addr.resize(num_page_table_entries);
if (has_attribute) {
attributes.resize(num_page_table_entries);
}
} }
} // namespace Common } // namespace Common

@ -4,6 +4,7 @@
#pragma once #pragma once
#include <atomic>
#include <tuple> #include <tuple>
#include "common/common_types.h" #include "common/common_types.h"
@ -20,10 +21,6 @@ enum class PageType : u8 {
/// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
/// invalidation /// invalidation
RasterizerCachedMemory, RasterizerCachedMemory,
/// Page is mapped to a I/O region. Writing and reading to this page is handled by functions.
Special,
/// Page is allocated for use.
Allocated,
}; };
struct SpecialRegion { struct SpecialRegion {
@ -48,6 +45,59 @@ struct SpecialRegion {
* mimics the way a real CPU page table works. * mimics the way a real CPU page table works.
*/ */
struct PageTable { struct PageTable {
/// Number of bits reserved for attribute tagging.
/// This can be at most the guaranteed alignment of the pointers in the page table.
static constexpr int ATTRIBUTE_BITS = 2;
/**
* Pair of host pointer and page type attribute.
* This uses the lower bits of a given pointer to store the attribute tag.
* Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
* call. In other words, they are guaranteed to be synchronized at all times.
*/
class PageInfo {
public:
/// Returns the page pointer
[[nodiscard]] u8* Pointer() const noexcept {
return ExtractPointer(raw.load(std::memory_order_relaxed));
}
/// Returns the page type attribute
[[nodiscard]] PageType Type() const noexcept {
return ExtractType(raw.load(std::memory_order_relaxed));
}
/// Returns the page pointer and attribute pair, extracted from the same atomic read
[[nodiscard]] std::pair<u8*, PageType> PointerType() const noexcept {
const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
}
/// Returns the raw representation of the page information.
/// Use ExtractPointer and ExtractType to unpack the value.
[[nodiscard]] uintptr_t Raw() const noexcept {
return raw.load(std::memory_order_relaxed);
}
/// Write a page pointer and type pair atomically
void Store(u8* pointer, PageType type) noexcept {
raw.store(reinterpret_cast<uintptr_t>(pointer) | static_cast<uintptr_t>(type));
}
/// Unpack a pointer from a page info raw representation
[[nodiscard]] static u8* ExtractPointer(uintptr_t raw) noexcept {
return reinterpret_cast<u8*>(raw & (~uintptr_t{0} << ATTRIBUTE_BITS));
}
/// Unpack a page type from a page info raw representation
[[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
}
private:
std::atomic<uintptr_t> raw;
};
PageTable(); PageTable();
~PageTable() noexcept; ~PageTable() noexcept;
@ -63,20 +113,16 @@ struct PageTable {
* *
* @param address_space_width_in_bits The address size width in bits. * @param address_space_width_in_bits The address size width in bits.
* @param page_size_in_bits The page size in bits. * @param page_size_in_bits The page size in bits.
* @param has_attribute Whether or not this page has any backing attributes.
*/ */
void Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, void Resize(size_t address_space_width_in_bits, size_t page_size_in_bits);
bool has_attribute);
/** /**
* Vector of memory pointers backing each page. An entry can only be non-null if the * Vector of memory pointers backing each page. An entry can only be non-null if the
* corresponding entry in the `attributes` vector is of type `Memory`. * corresponding attribute element is of type `Memory`.
*/ */
VirtualBuffer<u8*> pointers; VirtualBuffer<PageInfo> pointers;
VirtualBuffer<u64> backing_addr; VirtualBuffer<u64> backing_addr;
VirtualBuffer<PageType> attributes;
}; };
} // namespace Common } // namespace Common

@ -15,10 +15,12 @@ void FreeMemoryPages(void* base, std::size_t size) noexcept;
template <typename T> template <typename T>
class VirtualBuffer final { class VirtualBuffer final {
public: public:
static_assert( // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible
std::is_trivially_constructible_v<T>, // using std::atomic_ref once libc++ has support for it
"T must be trivially constructible, as non-trivial constructors will not be executed " // static_assert(
"with the current allocator"); // std::is_trivially_constructible_v<T>,
// "T must be trivially constructible, as non-trivial constructors will not be executed "
// "with the current allocator");
constexpr VirtualBuffer() = default; constexpr VirtualBuffer() = default;
explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} { explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} {

@ -133,6 +133,7 @@ std::shared_ptr<Dynarmic::A32::Jit> ARM_Dynarmic_32::MakeJit(Common::PageTable&
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>( config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>(
page_table.pointers.data()); page_table.pointers.data());
config.absolute_offset_page_table = true; config.absolute_offset_page_table = true;
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
config.only_detect_misalignment_via_page_table_on_page_boundary = true; config.only_detect_misalignment_via_page_table_on_page_boundary = true;

@ -152,6 +152,7 @@ std::shared_ptr<Dynarmic::A64::Jit> ARM_Dynarmic_64::MakeJit(Common::PageTable&
// Memory // Memory
config.page_table = reinterpret_cast<void**>(page_table.pointers.data()); config.page_table = reinterpret_cast<void**>(page_table.pointers.data());
config.page_table_address_space_bits = address_space_bits; config.page_table_address_space_bits = address_space_bits;
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.silently_mirror_page_table = false; config.silently_mirror_page_table = false;
config.absolute_offset_page_table = true; config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;

@ -265,7 +265,7 @@ ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_t
physical_memory_usage = 0; physical_memory_usage = 0;
memory_pool = pool; memory_pool = pool;
page_table_impl.Resize(address_space_width, PageBits, true); page_table_impl.Resize(address_space_width, PageBits);
return InitializeMemoryLayout(start, end); return InitializeMemoryLayout(start, end);
} }

@ -4,7 +4,6 @@
#include <algorithm> #include <algorithm>
#include <cstring> #include <cstring>
#include <mutex>
#include <optional> #include <optional>
#include <utility> #include <utility>
@ -68,21 +67,8 @@ struct Memory::Impl {
bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const { bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const {
const auto& page_table = process.PageTable().PageTableImpl(); const auto& page_table = process.PageTable().PageTableImpl();
const auto [pointer, type] = page_table.pointers[vaddr >> PAGE_BITS].PointerType();
const u8* const page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; return pointer != nullptr || type == Common::PageType::RasterizerCachedMemory;
if (page_pointer != nullptr) {
return true;
}
if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) {
return true;
}
if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) {
return false;
}
return false;
} }
bool IsValidVirtualAddress(VAddr vaddr) const { bool IsValidVirtualAddress(VAddr vaddr) const {
@ -100,17 +86,15 @@ struct Memory::Impl {
} }
u8* GetPointer(const VAddr vaddr) const { u8* GetPointer(const VAddr vaddr) const {
u8* const page_pointer{current_page_table->pointers[vaddr >> PAGE_BITS]}; const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (page_pointer) { if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
return page_pointer + vaddr; return pointer + vaddr;
} }
const auto type = Common::PageTable::PageInfo::ExtractType(raw_pointer);
if (current_page_table->attributes[vaddr >> PAGE_BITS] == if (type == Common::PageType::RasterizerCachedMemory) {
Common::PageType::RasterizerCachedMemory) {
return GetPointerFromRasterizerCachedMemory(vaddr); return GetPointerFromRasterizerCachedMemory(vaddr);
} }
return nullptr;
return {};
} }
u8 Read8(const VAddr addr) { u8 Read8(const VAddr addr) {
@ -222,7 +206,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -231,10 +216,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
const u8* const src_ptr =
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_buffer, src_ptr, copy_amount); std::memcpy(dest_buffer, src_ptr, copy_amount);
break; break;
} }
@ -268,7 +251,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -277,10 +261,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
const u8* const src_ptr =
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_buffer, src_ptr, copy_amount); std::memcpy(dest_buffer, src_ptr, copy_amount);
break; break;
} }
@ -320,7 +302,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -328,10 +311,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
u8* const dest_ptr =
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_ptr, src_buffer, copy_amount); std::memcpy(dest_ptr, src_buffer, copy_amount);
break; break;
} }
@ -364,7 +345,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -372,10 +354,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
u8* const dest_ptr =
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
std::memcpy(dest_ptr, src_buffer, copy_amount); std::memcpy(dest_ptr, src_buffer, copy_amount);
break; break;
} }
@ -414,7 +394,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -422,10 +403,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
u8* dest_ptr =
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
std::memset(dest_ptr, 0, copy_amount); std::memset(dest_ptr, 0, copy_amount);
break; break;
} }
@ -461,7 +440,8 @@ struct Memory::Impl {
std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
switch (page_table.attributes[page_index]) { const auto [pointer, type] = page_table.pointers[page_index].PointerType();
switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
LOG_ERROR(HW_Memory, LOG_ERROR(HW_Memory,
"Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@ -470,9 +450,8 @@ struct Memory::Impl {
break; break;
} }
case Common::PageType::Memory: { case Common::PageType::Memory: {
DEBUG_ASSERT(page_table.pointers[page_index]); DEBUG_ASSERT(pointer);
const u8* src_ptr = const u8* src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
WriteBlock(process, dest_addr, src_ptr, copy_amount); WriteBlock(process, dest_addr, src_ptr, copy_amount);
break; break;
} }
@ -498,34 +477,19 @@ struct Memory::Impl {
return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size); return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size);
} }
struct PageEntry {
u8* const pointer;
const Common::PageType attribute;
};
PageEntry SafePageEntry(std::size_t base) const {
std::lock_guard lock{rasterizer_cache_guard};
return {
.pointer = current_page_table->pointers[base],
.attribute = current_page_table->attributes[base],
};
}
void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
std::lock_guard lock{rasterizer_cache_guard};
if (vaddr == 0) { if (vaddr == 0) {
return; return;
} }
// Iterate over a contiguous CPU address space, which corresponds to the specified GPU // Iterate over a contiguous CPU address space, which corresponds to the specified GPU
// address space, marking the region as un/cached. The region is marked un/cached at a // address space, marking the region as un/cached. The region is marked un/cached at a
// granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size
// is different). This assumes the specified GPU address region is contiguous as well. // is different). This assumes the specified GPU address region is contiguous as well.
u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1; const u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1;
for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) { for (u64 i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) {
Common::PageType& page_type{current_page_table->attributes[vaddr >> PAGE_BITS]}; const Common::PageType page_type{
current_page_table->pointers[vaddr >> PAGE_BITS].Type()};
if (cached) { if (cached) {
// Switch page type to cached if now cached // Switch page type to cached if now cached
switch (page_type) { switch (page_type) {
@ -534,8 +498,8 @@ struct Memory::Impl {
// space, for example, a system module need not have a VRAM mapping. // space, for example, a system module need not have a VRAM mapping.
break; break;
case Common::PageType::Memory: case Common::PageType::Memory:
page_type = Common::PageType::RasterizerCachedMemory; current_page_table->pointers[vaddr >> PAGE_BITS].Store(
current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; nullptr, Common::PageType::RasterizerCachedMemory);
break; break;
case Common::PageType::RasterizerCachedMemory: case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common // There can be more than one GPU region mapped per CPU region, so it's common
@ -556,16 +520,16 @@ struct Memory::Impl {
// that this area is already unmarked as cached. // that this area is already unmarked as cached.
break; break;
case Common::PageType::RasterizerCachedMemory: { case Common::PageType::RasterizerCachedMemory: {
u8* pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)}; u8* const pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)};
if (pointer == nullptr) { if (pointer == nullptr) {
// It's possible that this function has been called while updating the // It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no // pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank. // longer exist, and we should just leave the pagetable entry blank.
page_type = Common::PageType::Unmapped; current_page_table->pointers[vaddr >> PAGE_BITS].Store(
nullptr, Common::PageType::Unmapped);
} else { } else {
current_page_table->pointers[vaddr >> PAGE_BITS] = current_page_table->pointers[vaddr >> PAGE_BITS].Store(
pointer - (vaddr & ~PAGE_MASK); pointer - (vaddr & ~PAGE_MASK), Common::PageType::Memory);
page_type = Common::PageType::Memory;
} }
break; break;
} }
@ -595,7 +559,7 @@ struct Memory::Impl {
auto& gpu = system.GPU(); auto& gpu = system.GPU();
for (u64 i = 0; i < size; i++) { for (u64 i = 0; i < size; i++) {
const auto page = base + i; const auto page = base + i;
if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) { if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) {
gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE); gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE);
} }
} }
@ -610,20 +574,18 @@ struct Memory::Impl {
"Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE); "Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE);
while (base != end) { while (base != end) {
page_table.attributes[base] = type; page_table.pointers[base].Store(nullptr, type);
page_table.pointers[base] = nullptr;
page_table.backing_addr[base] = 0; page_table.backing_addr[base] = 0;
base += 1; base += 1;
} }
} else { } else {
while (base != end) { while (base != end) {
page_table.pointers[base] = page_table.pointers[base].Store(
system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS); system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS), type);
page_table.attributes[base] = type;
page_table.backing_addr[base] = target - (base << PAGE_BITS); page_table.backing_addr[base] = target - (base << PAGE_BITS);
ASSERT_MSG(page_table.pointers[base], ASSERT_MSG(page_table.pointers[base].Pointer(),
"memory mapping base yield a nullptr within the table"); "memory mapping base yield a nullptr within the table");
base += 1; base += 1;
@ -646,21 +608,13 @@ struct Memory::Impl {
template <typename T> template <typename T>
T Read(const VAddr vaddr) { T Read(const VAddr vaddr) {
// Avoid adding any extra logic to this fast-path block // Avoid adding any extra logic to this fast-path block
if (const u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (const u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
T value; T value;
std::memcpy(&value, &pointer[vaddr], sizeof(T)); std::memcpy(&value, &pointer[vaddr], sizeof(T));
return value; return value;
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
// Otherwise, we need to grab the page with a lock, in case it is currently being modified
const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
if (entry.pointer) {
T value;
std::memcpy(&value, &entry.pointer[vaddr], sizeof(T));
return value;
}
switch (entry.attribute) {
case Common::PageType::Unmapped: case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
return 0; return 0;
@ -692,20 +646,12 @@ struct Memory::Impl {
template <typename T> template <typename T>
void Write(const VAddr vaddr, const T data) { void Write(const VAddr vaddr, const T data) {
// Avoid adding any extra logic to this fast-path block // Avoid adding any extra logic to this fast-path block
if (u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
std::memcpy(&pointer[vaddr], &data, sizeof(T)); std::memcpy(&pointer[vaddr], &data, sizeof(T));
return; return;
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
// Otherwise, we need to grab the page with a lock, in case it is currently being modified
const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
if (entry.pointer) {
// Memory was mapped, we are done
std::memcpy(&entry.pointer[vaddr], &data, sizeof(T));
return;
}
switch (entry.attribute) {
case Common::PageType::Unmapped: case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
static_cast<u32>(data), vaddr); static_cast<u32>(data), vaddr);
@ -726,15 +672,13 @@ struct Memory::Impl {
template <typename T> template <typename T>
bool WriteExclusive(const VAddr vaddr, const T data, const T expected) { bool WriteExclusive(const VAddr vaddr, const T data, const T expected) {
u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (page_pointer != nullptr) { if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
// NOTE: Avoid adding any extra logic to this fast-path block // NOTE: Avoid adding any extra logic to this fast-path block
auto* pointer = reinterpret_cast<volatile T*>(&page_pointer[vaddr]); const auto volatile_pointer = reinterpret_cast<volatile T*>(&pointer[vaddr]);
return Common::AtomicCompareAndSwap(pointer, data, expected); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
switch (type) {
case Common::PageType::Unmapped: case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
static_cast<u32>(data), vaddr); static_cast<u32>(data), vaddr);
@ -755,15 +699,13 @@ struct Memory::Impl {
} }
bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) { bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) {
u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
if (page_pointer != nullptr) { if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
// NOTE: Avoid adding any extra logic to this fast-path block // NOTE: Avoid adding any extra logic to this fast-path block
auto* pointer = reinterpret_cast<volatile u64*>(&page_pointer[vaddr]); const auto volatile_pointer = reinterpret_cast<volatile u64*>(&pointer[vaddr]);
return Common::AtomicCompareAndSwap(pointer, data, expected); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
} }
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
switch (type) {
case Common::PageType::Unmapped: case Common::PageType::Unmapped:
LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8, LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8,
static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr); static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr);
@ -783,7 +725,6 @@ struct Memory::Impl {
return true; return true;
} }
mutable std::mutex rasterizer_cache_guard;
Common::PageTable* current_page_table = nullptr; Common::PageTable* current_page_table = nullptr;
Core::System& system; Core::System& system;
}; };