Added vm::page_size constant

Added DECLARE_ENUM_CLASS_BITWISE_OPERATORS, fnv_1a_hasher, binary_equals utilities
This commit is contained in:
DHrpcs3 2016-02-28 17:33:41 +03:00
parent 20d6072e75
commit a748e7738a
6 changed files with 204 additions and 76 deletions

View file

@ -23,6 +23,29 @@ using s16 = std::int16_t;
using s32 = std::int32_t;
using s64 = std::int64_t;
#define DECLARE_ENUM_CLASS_BITWISE_OPERATORS(type) \
inline type operator |(type lhs, type rhs) \
{ \
return type(std::underlying_type_t<type>(lhs) | std::underlying_type_t<type>(rhs)); \
} \
inline type operator &(type lhs, type rhs) \
{ \
return type(std::underlying_type_t<type>(lhs) & std::underlying_type_t<type>(rhs)); \
} \
inline type& operator |=(type& lhs, type rhs) \
{ \
return lhs = lhs | rhs; \
} \
inline type& operator &=(type& lhs, type rhs) \
{ \
return lhs = lhs & rhs; \
} \
inline type operator ~(type lhs) \
{ \
return type(~std::underlying_type_t<type>(lhs)); \
} \
union alignas(2) f16
{
u16 _u16;
@ -47,10 +70,62 @@ union alignas(2) f16
using f32 = float;
using f64 = double;
template<std::size_t Size = sizeof(std::size_t)>
struct fnv_1;
template<>
struct fnv_1<8>
{
static const std::size_t offset_basis = 14695981039346656037ULL;
static const std::size_t prime = 1099511628211ULL;
};
struct fnv_1a_hasher
{
static std::size_t hash(const u8* raw, std::size_t size)
{
std::size_t result = fnv_1<>::offset_basis;
for (std::size_t byte = 0; byte < size; ++byte)
{
result ^= (std::size_t)raw[byte];
result *= fnv_1<>::prime;
}
return result;
}
template<typename Type>
static std::size_t hash(const Type& obj)
{
return hash((const u8*)&obj, sizeof(Type));
}
template<typename Type>
std::size_t operator()(const Type& obj) const
{
return hash(obj);
}
};
struct binary_equals
{
template<typename TypeA, typename TypeB>
bool operator()(const TypeA& a, const TypeB& b) const
{
static_assert(sizeof(TypeA) == sizeof(TypeB), "");
const void *a_ptr = reinterpret_cast<const void*>(std::addressof(a));
const void *b_ptr = reinterpret_cast<const void*>(std::addressof(b));
return a_ptr == b_ptr || memcmp(a_ptr, b_ptr, sizeof(TypeA)) == 0;
}
};
struct ignore
{
template<typename T>
ignore(T)
ignore(const T &)
{
}
};

View file

@ -85,7 +85,7 @@ namespace vm
u8* const g_base_addr = g_addr_set[0].get();
u8* const g_priv_addr = g_addr_set[1].get();
std::array<atomic_t<u8>, 0x100000000ull / 4096> g_pages{}; // information about every page
std::array<atomic_t<u8>, 0x100000000ull / page_size> g_pages{}; // information about every page
std::vector<std::shared_ptr<block_t>> g_locations; // memory locations
@ -170,7 +170,7 @@ namespace vm
const u64 align = 0x80000000ull >> cntlz32(size);
if (!size || !addr || size > 4096 || size != align || addr & (align - 1))
if (!size || !addr || size > page_size || size != align || addr & (align - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -316,7 +316,7 @@ namespace vm
{
const u64 align = 0x80000000ull >> cntlz32(size);
if (!size || !addr || size > 4096 || size != align || addr & (align - 1))
if (!size || !addr || size > page_size || size != align || addr & (align - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -372,9 +372,9 @@ namespace vm
{
#ifdef _WIN32
DWORD old;
if (!::VirtualProtect(vm::base(addr & ~0xfff), 4096, no_access ? PAGE_NOACCESS : PAGE_READONLY, &old))
if (!::VirtualProtect(vm::base(addr & ~0xfff), page_size, no_access ? PAGE_NOACCESS : PAGE_READONLY, &old))
#else
if (::mprotect(vm::base(addr & ~0xfff), 4096, no_access ? PROT_NONE : PROT_READ))
if (::mprotect(vm::base(addr & ~0xfff), page_size, no_access ? PROT_NONE : PROT_READ))
#endif
{
throw EXCEPTION("System failure (addr=0x%x)", addr);
@ -387,9 +387,9 @@ namespace vm
{
#ifdef _WIN32
DWORD old;
if (!::VirtualProtect(vm::base(addr & ~0xfff), 4096, PAGE_READWRITE, &old))
if (!::VirtualProtect(vm::base(addr & ~0xfff), page_size, PAGE_READWRITE, &old))
#else
if (::mprotect(vm::base(addr & ~0xfff), 4096, PROT_READ | PROT_WRITE))
if (::mprotect(vm::base(addr & ~0xfff), page_size, PROT_READ | PROT_WRITE))
#endif
{
throw EXCEPTION("System failure (addr=0x%x)", addr);
@ -424,7 +424,7 @@ namespace vm
const u64 align = 0x80000000ull >> cntlz32(size);
if (!size || !addr || size > 4096 || size != align || addr & (align - 1))
if (!size || !addr || size > page_size || size != align || addr & (align - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -460,7 +460,7 @@ namespace vm
const u64 align = 0x80000000ull >> cntlz32(size);
if (!size || !addr || size > 4096 || size != align || addr & (align - 1))
if (!size || !addr || size > page_size || size != align || addr & (align - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -545,7 +545,7 @@ namespace vm
const u64 align = 0x80000000ull >> cntlz32(size);
if (!size || !addr || size > 4096 || size != align || addr & (align - 1))
if (!size || !addr || size > page_size || size != align || addr & (align - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -586,16 +586,16 @@ namespace vm
void _page_map(u32 addr, u32 size, u8 flags)
{
if (!size || (size | addr) % 4096 || flags & page_allocated)
if (!size || (size | addr) & (page_size - 1) || flags & page_allocated)
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
if (g_pages[i])
{
throw EXCEPTION("Memory already mapped (addr=0x%x, size=0x%x, flags=0x%x, current_addr=0x%x)", addr, size, flags, i * 4096);
throw EXCEPTION("Memory already mapped (addr=0x%x, size=0x%x, flags=0x%x, current_addr=0x%x)", addr, size, flags, i * page_size);
}
}
@ -613,11 +613,11 @@ namespace vm
throw EXCEPTION("System failure (addr=0x%x, size=0x%x, flags=0x%x)", addr, size, flags);
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
if (g_pages[i].exchange(flags | page_allocated))
{
throw EXCEPTION("Concurrent access (addr=0x%x, size=0x%x, flags=0x%x, current_addr=0x%x)", addr, size, flags, i * 4096);
throw EXCEPTION("Concurrent access (addr=0x%x, size=0x%x, flags=0x%x, current_addr=0x%x)", addr, size, flags, i * page_size);
}
}
@ -628,7 +628,7 @@ namespace vm
{
std::lock_guard<reservation_mutex_t> lock(g_reservation_mutex);
if (!size || (size | addr) % 4096)
if (!size || (size | addr) & (page_size - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -637,7 +637,7 @@ namespace vm
flags_test |= page_allocated;
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
if ((g_pages[i] & flags_test) != (flags_test | page_allocated))
{
@ -650,9 +650,9 @@ namespace vm
return true;
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
_reservation_break(i * 4096);
_reservation_break(i * page_size);
const u8 f1 = g_pages[i]._or(flags_set & ~flags_inv) & (page_writable | page_readable);
g_pages[i]._and_not(flags_clear & ~flags_inv);
@ -660,16 +660,16 @@ namespace vm
if (f1 != f2)
{
void* real_addr = vm::base(i * 4096);
void* real_addr = vm::base(i * page_size);
#ifdef _WIN32
DWORD old;
auto protection = f2 & page_writable ? PAGE_READWRITE : (f2 & page_readable ? PAGE_READONLY : PAGE_NOACCESS);
if (!::VirtualProtect(real_addr, 4096, protection, &old))
if (!::VirtualProtect(real_addr, page_size, protection, &old))
#else
auto protection = f2 & page_writable ? PROT_WRITE | PROT_READ : (f2 & page_readable ? PROT_READ : PROT_NONE);
if (::mprotect(real_addr, 4096, protection))
if (::mprotect(real_addr, page_size, protection))
#endif
{
throw EXCEPTION("System failure (addr=0x%x, size=0x%x, flags_test=0x%x, flags_set=0x%x, flags_clear=0x%x)", addr, size, flags_test, flags_set, flags_clear);
@ -682,26 +682,26 @@ namespace vm
void _page_unmap(u32 addr, u32 size)
{
if (!size || (size | addr) % 4096)
if (!size || (size | addr) & (page_size - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
if ((g_pages[i] & page_allocated) == 0)
{
throw EXCEPTION("Memory not mapped (addr=0x%x, size=0x%x, current_addr=0x%x)", addr, size, i * 4096);
throw EXCEPTION("Memory not mapped (addr=0x%x, size=0x%x, current_addr=0x%x)", addr, size, i * page_size);
}
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
_reservation_break(i * 4096);
_reservation_break(i * page_size);
if (!(g_pages[i].exchange(0) & page_allocated))
{
throw EXCEPTION("Concurrent access (addr=0x%x, size=0x%x, current_addr=0x%x)", addr, size, i * 4096);
throw EXCEPTION("Concurrent access (addr=0x%x, size=0x%x, current_addr=0x%x)", addr, size, i * page_size);
}
}
@ -727,7 +727,7 @@ namespace vm
return false;
}
for (u32 i = addr / 4096; i <= (addr + size - 1) / 4096; i++)
for (u32 i = addr / page_size; i <= (addr + size - 1) / page_size; i++)
{
if ((g_pages[i] & page_allocated) == 0)
{
@ -794,7 +794,7 @@ namespace vm
bool block_t::try_alloc(u32 addr, u32 size)
{
// check if memory area is already mapped
for (u32 i = addr / 4096; i <= (addr + size - 1) / 4096; i++)
for (u32 i = addr / page_size; i <= (addr + size - 1) / page_size; i++)
{
if (g_pages[i])
{
@ -848,10 +848,10 @@ namespace vm
std::lock_guard<std::mutex> lock(m_mutex);
// align to minimal page size
size = ::align(size, 4096);
size = ::align(size, page_size);
// check alignment (it's page allocation, so passing small values there is just silly)
if (align < 4096 || align != (0x80000000u >> cntlz32(align)))
if (align < page_size || align != (0x80000000u >> cntlz32(align)))
{
throw EXCEPTION("Invalid alignment (size=0x%x, align=0x%x)", size, align);
}
@ -884,7 +884,7 @@ namespace vm
std::lock_guard<std::mutex> lock(m_mutex);
// align to minimal page size
size = ::align(size, 4096);
size = ::align(size, page_size);
// return if addr or size is invalid
if (!size || size > this->size || addr < this->addr || addr + size - 1 >= this->addr + this->size - 1)
@ -929,7 +929,7 @@ namespace vm
{
std::lock_guard<reservation_mutex_t> lock(g_reservation_mutex);
if (!size || (size | addr) % 4096)
if (!size || (size | addr) & (page_size - 1))
{
throw EXCEPTION("Invalid arguments (addr=0x%x, size=0x%x)", addr, size);
}
@ -947,11 +947,11 @@ namespace vm
}
}
for (u32 i = addr / 4096; i < addr / 4096 + size / 4096; i++)
for (u32 i = addr / page_size; i < addr / page_size + size / page_size; i++)
{
if (g_pages[i])
{
throw EXCEPTION("Unexpected pages allocated (current_addr=0x%x)", i * 4096);
throw EXCEPTION("Unexpected pages allocated (current_addr=0x%x)", i * page_size);
}
}

View file

@ -4,6 +4,8 @@
namespace vm
{
static constexpr u32 page_size = 0x1000;
extern u8* const g_base_addr;
extern u8* const g_priv_addr;
@ -328,7 +330,7 @@ namespace vm
void init();
}
namespace psv
{
template<typename T> inline to_le_t<T>* _ptr(u32 addr)

View file

@ -1512,6 +1512,5 @@ u64 GLGSRender::timestamp() const
bool GLGSRender::on_access_violation(u32 address, bool is_writing)
{
if (is_writing) return m_gl_texture_cache.mark_as_dirty(address);
return false;
}
return m_gl_texture_cache.sync_at((is_writing ? gl::cache_buffers::host : gl::cache_buffers::local), address);
}

View file

@ -21,27 +21,24 @@ namespace gl
bool texture_cache::lock_memory_region(u32 start, u32 size)
{
static const u32 memory_page_size = 4096;
start = start & ~(memory_page_size - 1);
size = (u32)align(size, memory_page_size);
start = start & ~(vm::page_size - 1);
size = (u32)align(size, vm::page_size);
return vm::page_protect(start, size, 0, 0, vm::page_writable);
}
bool texture_cache::unlock_memory_region(u32 start, u32 size)
{
static const u32 memory_page_size = 4096;
start = start & ~(memory_page_size - 1);
size = (u32)align(size, memory_page_size);
start = start & ~(vm::page_size - 1);
size = (u32)align(size, vm::page_size);
return vm::page_protect(start, size, 0, vm::page_writable, 0);
}
void texture_cache::lock_gl_object(cached_texture &obj)
{
static const u32 memory_page_size = 4096;
obj.protected_block_start = obj.data_addr & ~(memory_page_size - 1);
obj.protected_block_sz = (u32)align(obj.block_sz, memory_page_size);
obj.protected_block_start = obj.data_addr & ~(vm::page_size - 1);
obj.protected_block_sz = (u32)align(obj.block_sz, vm::page_size);
if (!lock_memory_region(obj.protected_block_start, obj.protected_block_sz))
LOG_ERROR(RSX, "lock_gl_object failed!");
@ -191,8 +188,8 @@ namespace gl
{
if (!rtt.data_addr || rtt.is_dirty) continue;
u32 rtt_aligned_base = ((u32)(rtt.data_addr)) & ~(4096 - 1);
u32 rtt_block_sz = align(rtt.block_sz, 4096);
u32 rtt_aligned_base = u32(rtt.data_addr) & ~(vm::page_size - 1);
u32 rtt_block_sz = align(rtt.block_sz, vm::page_size);
if (region_overlaps(rtt_aligned_base, (rtt_aligned_base + rtt_block_sz), base, base + size))
{
@ -403,15 +400,20 @@ namespace gl
gl_texture.set_id(real_id);
}
return;
}
else if (rtt)
{
LOG_NOTICE(RSX, "RTT texture for address 0x%X is dirty!", texaddr);
}
cached_texture *obj = nullptr;
if (!rtt)
{
obj = find_obj_for_params(texaddr, tex.width(), tex.height(), tex.mipmap());
}
if (obj && !obj->deleted)
{
@ -444,13 +446,16 @@ namespace gl
}
}
bool texture_cache::mark_as_dirty(u32 address)
bool texture_cache::mark_local_as_dirty_at(u32 address)
{
bool response = false;
for (cached_texture &tex : m_texture_cache)
{
if (!tex.locked) continue;
if (!tex.locked)
{
continue;
}
if (tex.protected_block_start <= address &&
tex.protected_block_sz > (address - tex.protected_block_start))
@ -465,27 +470,28 @@ namespace gl
}
}
if (response) return true;
for (cached_rtt &rtt : m_rtt_cache)
if (!response)
{
if (!rtt.data_addr || rtt.is_dirty) continue;
u32 rtt_aligned_base = ((u32)(rtt.data_addr)) & ~(4096 - 1);
u32 rtt_block_sz = align(rtt.block_sz, 4096);
if (rtt.locked && (u64)address >= rtt_aligned_base)
for (cached_rtt &rtt : m_rtt_cache)
{
u32 offset = address - rtt_aligned_base;
if (offset >= rtt_block_sz) continue;
if (!rtt.data_addr || rtt.is_dirty) continue;
LOG_NOTICE(RSX, "Dirty non-texture RTT FOUND! addr=0x%X", rtt.data_addr);
rtt.is_dirty = true;
u32 rtt_aligned_base = u32(rtt.data_addr) & ~(vm::page_size - 1);
u32 rtt_block_sz = align(rtt.block_sz, vm::page_size);
unlock_memory_region(rtt_aligned_base, rtt_block_sz);
rtt.locked = false;
if (rtt.locked && (u64)address >= rtt_aligned_base)
{
u32 offset = address - rtt_aligned_base;
if (offset >= rtt_block_sz) continue;
response = true;
LOG_NOTICE(RSX, "Dirty non-texture RTT FOUND! addr=0x%X", rtt.data_addr);
rtt.is_dirty = true;
unlock_memory_region(rtt_aligned_base, rtt_block_sz);
rtt.locked = false;
response = true;
}
}
}
@ -547,9 +553,9 @@ namespace gl
if (base < obj.data_addr)
invalid.block_base = obj.protected_block_start;
else
invalid.block_base = obj.protected_block_start + obj.protected_block_sz - 4096;
invalid.block_base = obj.protected_block_start + obj.protected_block_sz - vm::page_size;
invalid.block_sz = 4096;
invalid.block_sz = vm::page_size;
unlock_memory_region(invalid.block_base, invalid.block_sz);
result.push_back(invalid);
}
@ -558,9 +564,9 @@ namespace gl
return result;
}
void texture_cache::lock_invalidated_ranges(std::vector<invalid_cache_area> invalid)
void texture_cache::lock_invalidated_ranges(const std::vector<invalid_cache_area> &invalid)
{
for (invalid_cache_area area : invalid)
for (const invalid_cache_area &area : invalid)
{
lock_memory_region(area.block_base, area.block_sz);
}
@ -603,4 +609,21 @@ namespace gl
//No valid object found in cache
return false;
}
bool texture_cache::sync_at(cache_buffers buffers, u32 address)
{
bool result = false;
if ((buffers & cache_buffers::host) != cache_buffers::none)
{
result = mark_local_as_dirty_at(address);
}
if ((buffers & cache_buffers::local) != cache_buffers::none)
{
//TODO
}
return result;
}
}

View file

@ -6,6 +6,33 @@
namespace gl
{
enum class cache_access
{
none,
read = 1 << 0,
write = 1 << 1,
read_write = read | write,
};
enum class cache_buffers
{
none = 0,
host = 1 << 0,
local = 1 << 1,
};
enum class cache_entry_state
{
invalidated = 0,
local_synchronized = 1 << 0,
host_synchronized = 1 << 1,
synchronized = local_synchronized | host_synchronized,
};
DECLARE_ENUM_CLASS_BITWISE_OPERATORS(cache_access);
DECLARE_ENUM_CLASS_BITWISE_OPERATORS(cache_buffers);
DECLARE_ENUM_CLASS_BITWISE_OPERATORS(cache_entry_state);
struct cached_texture
{
u32 gl_id;
@ -73,11 +100,13 @@ namespace gl
void update_frame_ctr();
void initialize_rtt_cache();
void upload_texture(int index, rsx::texture &tex, rsx::gl::texture &gl_texture);
bool mark_as_dirty(u32 address);
bool mark_local_as_dirty_at(u32 address);
void save_render_target(u32 texaddr, u32 range, gl::texture &gl_texture);
std::vector<invalid_cache_area> find_and_invalidate_in_range(u32 base, u32 limit);
void lock_invalidated_ranges(std::vector<invalid_cache_area> invalid);
void lock_invalidated_ranges(const std::vector<invalid_cache_area> &invalid);
void remove_in_range(u32 texaddr, u32 range);
bool explicit_writeback(gl::texture &tex, const u32 address, const u32 pitch);
bool sync_at(cache_buffers buffer, u32 address);
};
}