diff --git a/Utilities/GNU.h b/Utilities/GNU.h index d4eb6cb103..3cf5d606c5 100644 --- a/Utilities/GNU.h +++ b/Utilities/GNU.h @@ -46,18 +46,21 @@ void strcpy_trunc(char(&dst)[size], const char(&src)[rsize]) #define _CRT_ALIGN(x) __attribute__((aligned(x))) #define InterlockedCompareExchange(ptr,new_val,old_val) __sync_val_compare_and_swap(ptr,old_val,new_val) #define InterlockedExchange(ptr, value) __sync_lock_test_and_set(ptr, value) +#define InterlockedOr(ptr, value) __sync_fetch_and_or(ptr, value) +#define InterlockedAnd(ptr, value) __sync_fetch_and_and(ptr, value) +#define InterlockedXor(ptr, value) __sync_fetch_and_xor(ptr, value) -inline int64_t InterlockedOr64(volatile int64_t *dest, int64_t val) -{ - int64_t olderval; - int64_t oldval = *dest; - do - { - olderval = oldval; - oldval = __sync_val_compare_and_swap(dest, olderval | val, olderval); - } while (olderval != oldval); - return oldval; -} +//inline int64_t InterlockedOr64(volatile int64_t *dest, int64_t val) +//{ +// int64_t olderval; +// int64_t oldval = *dest; +// do +// { +// olderval = oldval; +// oldval = __sync_val_compare_and_swap(dest, olderval | val, olderval); +// } while (olderval != oldval); +// return oldval; +//} inline uint64_t __umulh(uint64_t a, uint64_t b) { @@ -112,3 +115,36 @@ static __forceinline uint64_t InterlockedExchange(volatile uint64_t* dest, uint6 return _InterlockedExchange64((volatile long long*)dest, value); } #endif + +#ifndef InterlockedOr +static __forceinline uint32_t InterlockedOr(volatile uint32_t* dest, uint32_t value) +{ + return _InterlockedOr((volatile long*)dest, value); +} +static __forceinline uint64_t InterlockedOr(volatile uint64_t* dest, uint64_t value) +{ + return _InterlockedOr64((volatile long long*)dest, value); +} +#endif + +#ifndef InterlockedAnd +static __forceinline uint32_t InterlockedAnd(volatile uint32_t* dest, uint32_t value) +{ + return _InterlockedAnd((volatile long*)dest, value); +} +static __forceinline uint64_t InterlockedAnd(volatile uint64_t* dest, uint64_t value) +{ + return _InterlockedAnd64((volatile long long*)dest, value); +} +#endif + +#ifndef InterlockedXor +static __forceinline uint32_t InterlockedXor(volatile uint32_t* dest, uint32_t value) +{ + return _InterlockedXor((volatile long*)dest, value); +} +static __forceinline uint64_t InterlockedXor(volatile uint64_t* dest, uint64_t value) +{ + return _InterlockedXor64((volatile long long*)dest, value); +} +#endif diff --git a/rpcs3/Emu/Cell/SPUThread.h b/rpcs3/Emu/Cell/SPUThread.h index 0e6070d0bf..e33291cf8c 100644 --- a/rpcs3/Emu/Cell/SPUThread.h +++ b/rpcs3/Emu/Cell/SPUThread.h @@ -358,7 +358,7 @@ public: } else { - InterlockedOr64((volatile s64*)m_indval, ((u64)value << 32) | 1); + InterlockedOr((volatile u64*)m_indval, ((u64)value << 32) | 1); } } diff --git a/rpcs3/Emu/Memory/vm_atomic.h b/rpcs3/Emu/Memory/vm_atomic.h index dba8a6af84..1b74b898fb 100644 --- a/rpcs3/Emu/Memory/vm_atomic.h +++ b/rpcs3/Emu/Memory/vm_atomic.h @@ -93,6 +93,42 @@ namespace vm } } + __forceinline const T _or(const T right) volatile + { + const atomic_type res = InterlockedOr((volatile atomic_type*)&data, (atomic_type&)right); + return (T&)res; + } + + __forceinline const T _and(const T right) volatile + { + const atomic_type res = InterlockedAnd((volatile atomic_type*)&data, (atomic_type&)right); + return (T&)res; + } + + __forceinline const T _xor(const T right) volatile + { + const atomic_type res = InterlockedXor((volatile atomic_type*)&data, (atomic_type&)right); + return (T&)res; + } + + __forceinline const T operator |= (const T right) volatile + { + const atomic_type res = InterlockedOr((volatile atomic_type*)&data, (atomic_type&)right) | (atomic_type&)right; + return (T&)res; + } + + __forceinline const T operator &= (const T right) volatile + { + const atomic_type res = InterlockedAnd((volatile atomic_type*)&data, (atomic_type&)right) & (atomic_type&)right; + return (T&)res; + } + + __forceinline const T operator ^= (const T right) volatile + { + const atomic_type res = InterlockedXor((volatile atomic_type*)&data, (atomic_type&)right) ^ (atomic_type&)right; + return (T&)res; + } + }; template struct atomic_le : public _atomic_base diff --git a/rpcs3/Emu/SysCalls/Modules/cellSync.cpp b/rpcs3/Emu/SysCalls/Modules/cellSync.cpp index a832c15dfa..47893dea43 100644 --- a/rpcs3/Emu/SysCalls/Modules/cellSync.cpp +++ b/rpcs3/Emu/SysCalls/Modules/cellSync.cpp @@ -54,9 +54,9 @@ s32 cellSyncMutexLock(vm::ptr mutex) // prx: increase m_acq and remember its old value be_t order; - mutex->data.atomic_op([&order](CellSyncMutex::data_t& _mutex) + mutex->data.atomic_op([&order](CellSyncMutex::data_t& mutex) { - order = _mutex.m_acq++; + order = mutex.m_acq++; }); // prx: wait until this old value is equal to m_rel @@ -89,9 +89,9 @@ s32 cellSyncMutexTryLock(vm::ptr mutex) } // prx: exit if m_acq and m_rel are not equal, increase m_acq - return mutex->data.atomic_op(CELL_OK, [](CellSyncMutex::data_t& _mutex) -> s32 + return mutex->data.atomic_op(CELL_OK, [](CellSyncMutex::data_t& mutex) -> s32 { - if (_mutex.m_acq++ != _mutex.m_rel) + if (mutex.m_acq++ != mutex.m_rel) { return CELL_SYNC_ERROR_BUSY; } @@ -113,9 +113,9 @@ s32 cellSyncMutexUnlock(vm::ptr mutex) } mutex->data.read_sync(); - mutex->data.atomic_op([](CellSyncMutex::data_t& _mutex) + mutex->data.atomic_op([](CellSyncMutex::data_t& mutex) { - _mutex.m_rel++; + mutex.m_rel++; }); return CELL_OK; } @@ -147,6 +147,24 @@ s32 cellSyncBarrierInitialize(vm::ptr barrier, u16 total_count) return syncBarrierInitialize(barrier, total_count); } +s32 syncBarrierTryNotifyOp(CellSyncBarrier::data_t& barrier) +{ + // prx: extract m_value (repeat if < 0), increase, compare with second s16, set sign bit if equal, insert it back + s16 value = (s16)barrier.m_value; + if (value < 0) + { + return CELL_SYNC_ERROR_BUSY; + } + + value++; + if (value == (s16)barrier.m_count) + { + value |= 0x8000; + } + barrier.m_value = value; + return CELL_OK; +}; + s32 cellSyncBarrierNotify(vm::ptr barrier) { cellSync->Log("cellSyncBarrierNotify(barrier_addr=0x%x)", barrier.addr()); @@ -160,35 +178,16 @@ s32 cellSyncBarrierNotify(vm::ptr barrier) return CELL_SYNC_ERROR_ALIGN; } - // prx: sync, extract m_value, repeat if < 0, increase, compare with second s16, set sign bit if equal, insert it back barrier->data.read_sync(); - - while (true) + while (barrier->data.atomic_op(CELL_OK, syncBarrierTryNotifyOp)) { - const auto old = barrier->data.read_relaxed(); - auto _barrier = old; - - s16 value = (s16)_barrier.m_value; - if (value < 0) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack + if (Emu.IsStopped()) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack - if (Emu.IsStopped()) - { - cellSync->Warning("cellSyncBarrierNotify(barrier_addr=0x%x) aborted", barrier.addr()); - return CELL_OK; - } - continue; + cellSync->Warning("cellSyncBarrierNotify(barrier_addr=0x%x) aborted", barrier.addr()); + return CELL_OK; } - - value++; - if (value == (s16)_barrier.m_count) - { - value |= 0x8000; - } - _barrier.m_value = value; - if (barrier->data.compare_and_swap_test(old, _barrier)) break; } - return CELL_OK; } @@ -206,29 +205,24 @@ s32 cellSyncBarrierTryNotify(vm::ptr barrier) } barrier->data.read_sync(); + return barrier->data.atomic_op(CELL_OK, syncBarrierTryNotifyOp); +} - while (true) +s32 syncBarrierTryWaitOp(CellSyncBarrier::data_t& barrier) +{ + // prx: extract m_value (repeat if >= 0), decrease it, set 0 if == 0x8000, insert it back + s16 value = (s16)barrier.m_value; + if (value >= 0) { - const auto old = barrier->data.read_relaxed(); - auto _barrier = old; - - s16 value = (s16)_barrier.m_value; - if (value >= 0) - { - value++; - if (value == (s16)_barrier.m_count) - { - value |= 0x8000; - } - _barrier.m_value = value; - if (barrier->data.compare_and_swap_test(old, _barrier)) break; - } - else - { - if (barrier->data.compare_and_swap_test(old, _barrier)) return CELL_SYNC_ERROR_BUSY; - } + return CELL_SYNC_ERROR_BUSY; } + value--; + if (value == (s16)0x8000) + { + value = 0; + } + barrier.m_value = value; return CELL_OK; } @@ -245,35 +239,16 @@ s32 cellSyncBarrierWait(vm::ptr barrier) return CELL_SYNC_ERROR_ALIGN; } - // prx: sync, extract m_value (repeat if >= 0), decrease it, set 0 if == 0x8000, insert it back barrier->data.read_sync(); - - while (true) + while (barrier->data.atomic_op(CELL_OK, syncBarrierTryWaitOp)) { - const auto old = barrier->data.read_relaxed(); - auto _barrier = old; - - s16 value = (s16)_barrier.m_value; - if (value >= 0) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack + if (Emu.IsStopped()) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack - if (Emu.IsStopped()) - { - cellSync->Warning("cellSyncBarrierWait(barrier_addr=0x%x) aborted", barrier.addr()); - return CELL_OK; - } - continue; + cellSync->Warning("cellSyncBarrierWait(barrier_addr=0x%x) aborted", barrier.addr()); + return CELL_OK; } - - value--; - if (value == (s16)0x8000) - { - value = 0; - } - _barrier.m_value = value; - if (barrier->data.compare_and_swap_test(old, _barrier)) break; } - return CELL_OK; } @@ -291,28 +266,7 @@ s32 cellSyncBarrierTryWait(vm::ptr barrier) } barrier->data.read_sync(); - - while (true) - { - const auto old = barrier->data.read_relaxed(); - auto _barrier = old; - - s16 value = (s16)_barrier.m_value; - if (value >= 0) - { - return CELL_SYNC_ERROR_BUSY; - } - - value--; - if (value == (s16)0x8000) - { - value = 0; - } - _barrier.m_value = value; - if (barrier->data.compare_and_swap_test(old, _barrier)) break; - } - - return CELL_OK; + return barrier->data.atomic_op(CELL_OK, syncBarrierTryWaitOp); } s32 syncRwmInitialize(vm::ptr rwm, vm::ptr buffer, u32 buffer_size) @@ -331,10 +285,9 @@ s32 syncRwmInitialize(vm::ptr rwm, vm::ptr buffer, u32 buffer } // prx: zeroize first u16 and second u16, write buffer_size in second u32, write buffer_addr in second u64 and sync - rwm->m_data() = 0; rwm->m_size = buffer_size; rwm->m_buffer = buffer; - InterlockedCompareExchange(&rwm->m_data(), 0, 0); + rwm->data.exchange({}); return CELL_OK; } @@ -345,6 +298,27 @@ s32 cellSyncRwmInitialize(vm::ptr rwm, vm::ptr buffer, u32 bu return syncRwmInitialize(rwm, buffer, buffer_size); } +s32 syncRwmTryReadBeginOp(CellSyncRwm::data_t& rwm) +{ + if (rwm.m_writers.ToBE()) + { + return CELL_SYNC_ERROR_BUSY; + } + rwm.m_readers++; + return CELL_OK; +} + +s32 syncRwmReadEndOp(CellSyncRwm::data_t& rwm) +{ + if (!rwm.m_readers.ToBE()) + { + cellSync->Error("syncRwmReadEndOp(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", Memory.RealToVirtualAddr(&rwm), (u16)rwm.m_writers); + return CELL_SYNC_ERROR_ABORT; + } + rwm.m_readers--; + return CELL_OK; +} + s32 cellSyncRwmRead(vm::ptr rwm, vm::ptr buffer) { cellSync->Log("cellSyncRwmRead(rwm_addr=0x%x, buffer_addr=0x%x)", rwm.addr(), buffer.addr()); @@ -358,48 +332,22 @@ s32 cellSyncRwmRead(vm::ptr rwm, vm::ptr buffer) return CELL_SYNC_ERROR_ALIGN; } - // prx: atomically load first u32, repeat until second u16 == 0, increase first u16 and sync - while (true) + // prx: increase m_readers, wait until m_writers is zero + while (rwm->data.atomic_op(CELL_OK, syncRwmTryReadBeginOp)) { - const u32 old_data = rwm->m_data(); - CellSyncRwm new_rwm; - new_rwm.m_data() = old_data; - - if (new_rwm.m_writers.ToBE()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack + if (Emu.IsStopped()) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack - if (Emu.IsStopped()) - { - cellSync->Warning("cellSyncRwmRead(rwm_addr=0x%x) aborted", rwm.addr()); - return CELL_OK; - } - continue; + cellSync->Warning("cellSyncRwmRead(rwm_addr=0x%x) aborted", rwm.addr()); + return CELL_OK; } - - new_rwm.m_readers++; - if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break; } // copy data to buffer_addr memcpy(buffer.get_ptr(), rwm->m_buffer.get_ptr(), (u32)rwm->m_size); - // prx: load first u32, return 0x8041010C if first u16 == 0, atomically decrease it - while (true) - { - const u32 old_data = rwm->m_data(); - CellSyncRwm new_rwm; - new_rwm.m_data() = old_data; - - if (!new_rwm.m_readers.ToBE()) - { - cellSync->Error("cellSyncRwmRead(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", rwm.addr(), (u16)new_rwm.m_writers); - return CELL_SYNC_ERROR_ABORT; - } - - new_rwm.m_readers--; - if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break; - } - return CELL_OK; + // prx: decrease m_readers (return 0x8041010C if already zero) + return rwm->data.atomic_op(CELL_OK, syncRwmReadEndOp); } s32 cellSyncRwmTryRead(vm::ptr rwm, vm::ptr buffer) @@ -415,38 +363,22 @@ s32 cellSyncRwmTryRead(vm::ptr rwm, vm::ptr buffer) return CELL_SYNC_ERROR_ALIGN; } - while (true) + if (s32 res = rwm->data.atomic_op(CELL_OK, syncRwmTryReadBeginOp)) { - const u32 old_data = rwm->m_data(); - CellSyncRwm new_rwm; - new_rwm.m_data() = old_data; - - if (new_rwm.m_writers.ToBE()) - { - return CELL_SYNC_ERROR_BUSY; - } - - new_rwm.m_readers++; - if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break; + return res; } - memcpy(buffer.get_ptr(), rwm->m_buffer.get_ptr(), (u32)rwm->m_size); - while (true) + return rwm->data.atomic_op(CELL_OK, syncRwmReadEndOp); +} + +s32 syncRwmTryWriteBeginOp(CellSyncRwm::data_t& rwm) +{ + if (rwm.m_writers.ToBE()) { - const u32 old_data = rwm->m_data(); - CellSyncRwm new_rwm; - new_rwm.m_data() = old_data; - - if (!new_rwm.m_readers.ToBE()) - { - cellSync->Error("cellSyncRwmRead(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", rwm.addr(), (u16)new_rwm.m_writers); - return CELL_SYNC_ERROR_ABORT; - } - - new_rwm.m_readers--; - if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break; + return CELL_SYNC_ERROR_BUSY; } + rwm.m_writers = 1; return CELL_OK; } @@ -463,30 +395,18 @@ s32 cellSyncRwmWrite(vm::ptr rwm, vm::ptr buffer) return CELL_SYNC_ERROR_ALIGN; } - // prx: atomically compare second u16 (m_writers) with 0, repeat if not 0, set 1, sync - while (true) + while (rwm->data.atomic_op(CELL_OK, syncRwmTryWriteBeginOp)) { - const u32 old_data = rwm->m_data(); - CellSyncRwm new_rwm; - new_rwm.m_data() = old_data; - - if (new_rwm.m_writers.ToBE()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack + if (Emu.IsStopped()) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack - if (Emu.IsStopped()) - { - cellSync->Warning("cellSyncRwmWrite(rwm_addr=0x%x) aborted (I)", rwm.addr()); - return CELL_OK; - } - continue; + cellSync->Warning("cellSyncRwmWrite(rwm_addr=0x%x) aborted (I)", rwm.addr()); + return CELL_OK; } - - new_rwm.m_writers = 1; - if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break; } // prx: wait until m_readers == 0 - while (rwm->m_readers.ToBE()) + while (rwm->data.read_relaxed().m_readers.ToBE()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack if (Emu.IsStopped()) @@ -500,8 +420,7 @@ s32 cellSyncRwmWrite(vm::ptr rwm, vm::ptr buffer) memcpy(rwm->m_buffer.get_ptr(), buffer.get_ptr(), (u32)rwm->m_size); // prx: sync and zeroize m_readers and m_writers - InterlockedCompareExchange(&rwm->m_data(), 0, 0); - rwm->m_data() = 0; + rwm->data.exchange({}); return CELL_OK; } @@ -518,15 +437,17 @@ s32 cellSyncRwmTryWrite(vm::ptr rwm, vm::ptr buffer) return CELL_SYNC_ERROR_ALIGN; } - // prx: compare m_readers | m_writers with 0, return busy if not zero, set m_writers to 1 - if (InterlockedCompareExchange(&rwm->m_data(), se32(1), 0) != 0) return CELL_SYNC_ERROR_BUSY; + // prx: compare m_readers | m_writers with 0, return if not zero, set m_writers to 1 + if (!rwm->data.compare_and_swap_test({}, {be_t::make(0), be_t::make(1)})) + { + return CELL_SYNC_ERROR_BUSY; + } // prx: copy data from buffer_addr memcpy(rwm->m_buffer.get_ptr(), buffer.get_ptr(), (u32)rwm->m_size); // prx: sync and zeroize m_readers and m_writers - InterlockedCompareExchange(&rwm->m_data(), 0, 0); - rwm->m_data() = 0; + rwm->data.exchange({}); return CELL_OK; } diff --git a/rpcs3/Emu/SysCalls/Modules/cellSync.h b/rpcs3/Emu/SysCalls/Modules/cellSync.h index 6d2306469a..15f94e46e0 100644 --- a/rpcs3/Emu/SysCalls/Modules/cellSync.h +++ b/rpcs3/Emu/SysCalls/Modules/cellSync.h @@ -57,23 +57,30 @@ static_assert(sizeof(CellSyncBarrier) == 4, "CellSyncBarrier: wrong size"); struct CellSyncRwm { - be_t m_readers; - be_t m_writers; + struct data_t + { + be_t m_readers; + be_t m_writers; + }; + + vm::atomic data; be_t m_size; vm::bptr m_buffer; - - volatile u32& m_data() - { - return *reinterpret_cast(this); - }; }; static_assert(sizeof(CellSyncRwm) == 16, "CellSyncRwm: wrong size"); struct CellSyncQueue { + struct data_t + { + be_t m_v1; + be_t m_v2; + }; + be_t m_v1; be_t m_v2; + //vm::atomic data; be_t m_size; be_t m_depth; vm::bptr m_buffer;