From 2226125728a49c51a9eea74fc84c284373bd33ed Mon Sep 17 00:00:00 2001 From: Nekotekina Date: Sun, 9 Sep 2018 12:59:24 +0300 Subject: [PATCH] atomic_t<>: remove inline assignment Add atomic_op/fetch_op overloads with template argument (may be removed in future) Remove args... in atomic_op (capturing lambda is preferred) --- Utilities/Atomic.h | 80 ++++++++++++++++++++++++--- rpcs3/Emu/Cell/Modules/cellSync.cpp | 70 ++++++++++++++--------- rpcs3/Emu/Cell/Modules/cellSync.h | 42 +++++++------- rpcs3/Emu/Cell/lv2/sys_event_flag.cpp | 15 ++++- 4 files changed, 149 insertions(+), 58 deletions(-) diff --git a/Utilities/Atomic.h b/Utilities/Atomic.h index a9232ecf55..e43c472125 100644 --- a/Utilities/Atomic.h +++ b/Utilities/Atomic.h @@ -618,9 +618,11 @@ public: while (true) { + _new = old; + if constexpr (std::is_void_v) { - std::invoke(func, (_new = old)); + std::invoke(std::forward(func), _new); if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) { @@ -629,7 +631,7 @@ public: } else { - RT ret = std::invoke(func, (_new = old)); + RT ret = std::invoke(std::forward(func), _new); if (LIKELY(!ret || atomic_storage::compare_exchange(m_data, old, _new))) { @@ -639,17 +641,50 @@ public: } } - // Atomic operation; returns function result value (TODO: remove args) - template > - RT atomic_op(F&& func, const Args&... args) + // fetch_op overload with function (invokable) provided as a template parameter + template > + std::conditional_t, type, std::pair> fetch_op() { type _new, old = atomic_storage::load(m_data); while (true) { + _new = old; + if constexpr (std::is_void_v) { - std::invoke(func, (_new = old), args...); + std::invoke(F, _new); + + if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) + { + return old; + } + } + else + { + RT ret = std::invoke(F, _new); + + if (LIKELY(!ret || atomic_storage::compare_exchange(m_data, old, _new))) + { + return {old, std::move(ret)}; + } + } + } + } + + // Atomic operation; returns function result value, function is the lambda + template > + RT atomic_op(F&& func) + { + type _new, old = atomic_storage::load(m_data); + + while (true) + { + _new = old; + + if constexpr (std::is_void_v) + { + std::invoke(std::forward(func), _new); if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) { @@ -658,7 +693,38 @@ public: } else { - RT result = std::invoke(func, (_new = old), args...); + RT result = std::invoke(std::forward(func), _new); + + if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) + { + return result; + } + } + } + } + + // atomic_op overload with function (invokable) provided as a template parameter + template > + RT atomic_op() + { + type _new, old = atomic_storage::load(m_data); + + while (true) + { + _new = old; + + if constexpr (std::is_void_v) + { + std::invoke(F, _new); + + if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) + { + return; + } + } + else + { + RT result = std::invoke(F, _new); if (LIKELY(atomic_storage::compare_exchange(m_data, old, _new))) { diff --git a/rpcs3/Emu/Cell/Modules/cellSync.cpp b/rpcs3/Emu/Cell/Modules/cellSync.cpp index 307edba4c0..50d5d6aca9 100644 --- a/rpcs3/Emu/Cell/Modules/cellSync.cpp +++ b/rpcs3/Emu/Cell/Modules/cellSync.cpp @@ -74,7 +74,7 @@ error_code cellSyncMutexLock(ppu_thread& ppu, vm::ptr mutex) } // Increase acq value and remember its old value - const auto order = mutex->ctrl.atomic_op(&CellSyncMutex::lock_begin); + const auto order = mutex->ctrl.atomic_op<&CellSyncMutex::Counter::lock_begin>(); // Wait until rel value is equal to old acq value while (mutex->ctrl.load().rel != order) @@ -101,7 +101,7 @@ error_code cellSyncMutexTryLock(vm::ptr mutex) return CELL_SYNC_ERROR_ALIGN; } - if (!mutex->ctrl.atomic_op(&CellSyncMutex::try_lock)) + if (!mutex->ctrl.atomic_op<&CellSyncMutex::Counter::try_lock>()) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -123,7 +123,7 @@ error_code cellSyncMutexUnlock(vm::ptr mutex) return CELL_SYNC_ERROR_ALIGN; } - mutex->ctrl.atomic_op(&CellSyncMutex::unlock); + mutex->ctrl.atomic_op<&CellSyncMutex::Counter::unlock>(); return CELL_OK; } @@ -167,7 +167,7 @@ error_code cellSyncBarrierNotify(ppu_thread& ppu, vm::ptr barri return CELL_SYNC_ERROR_ALIGN; } - while (!barrier->ctrl.atomic_op(&CellSyncBarrier::try_notify)) + while (!barrier->ctrl.atomic_op<&CellSyncBarrier::try_notify>()) { ppu.test_state(); } @@ -191,7 +191,7 @@ error_code cellSyncBarrierTryNotify(vm::ptr barrier) _mm_mfence(); - if (!barrier->ctrl.atomic_op(&CellSyncBarrier::try_notify)) + if (!barrier->ctrl.atomic_op<&CellSyncBarrier::try_notify>()) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -215,7 +215,7 @@ error_code cellSyncBarrierWait(ppu_thread& ppu, vm::ptr barrier _mm_mfence(); - while (!barrier->ctrl.atomic_op(&CellSyncBarrier::try_wait)) + while (!barrier->ctrl.atomic_op<&CellSyncBarrier::try_wait>()) { ppu.test_state(); } @@ -239,7 +239,7 @@ error_code cellSyncBarrierTryWait(vm::ptr barrier) _mm_mfence(); - if (!barrier->ctrl.atomic_op(&CellSyncBarrier::try_wait)) + if (!barrier->ctrl.atomic_op<&CellSyncBarrier::try_wait>()) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -291,7 +291,7 @@ error_code cellSyncRwmRead(ppu_thread& ppu, vm::ptr rwm, vm::ptrctrl.atomic_op(&CellSyncRwm::try_read_begin)) + while (!rwm->ctrl.atomic_op<&CellSyncRwm::try_read_begin>()) { ppu.test_state(); } @@ -300,7 +300,7 @@ error_code cellSyncRwmRead(ppu_thread& ppu, vm::ptr rwm, vm::ptrbuffer.get_ptr(), rwm->size); // decrease `readers`, return error if already zero - if (!rwm->ctrl.atomic_op(&CellSyncRwm::try_read_end)) + if (!rwm->ctrl.atomic_op<&CellSyncRwm::try_read_end>()) { return CELL_SYNC_ERROR_ABORT; } @@ -323,7 +323,7 @@ error_code cellSyncRwmTryRead(vm::ptr rwm, vm::ptr buffer) } // increase `readers` if `writers` is zero - if (!rwm->ctrl.atomic_op(&CellSyncRwm::try_read_begin)) + if (!rwm->ctrl.atomic_op<&CellSyncRwm::try_read_begin>()) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -332,7 +332,7 @@ error_code cellSyncRwmTryRead(vm::ptr rwm, vm::ptr buffer) std::memcpy(buffer.get_ptr(), rwm->buffer.get_ptr(), rwm->size); // decrease `readers`, return error if already zero - if (!rwm->ctrl.atomic_op(&CellSyncRwm::try_read_end)) + if (!rwm->ctrl.atomic_op<&CellSyncRwm::try_read_end>()) { return CELL_SYNC_ERROR_ABORT; } @@ -355,7 +355,7 @@ error_code cellSyncRwmWrite(ppu_thread& ppu, vm::ptr rwm, vm::cptr< } // wait until `writers` is zero, set to 1 - while (!rwm->ctrl.atomic_op(&CellSyncRwm::try_write_begin)) + while (!rwm->ctrl.atomic_op<&CellSyncRwm::try_write_begin>()) { ppu.test_state(); } @@ -457,7 +457,10 @@ error_code cellSyncQueuePush(ppu_thread& ppu, vm::ptr queue, vm:: u32 position; - while (!queue->ctrl.atomic_op(&CellSyncQueue::try_push_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_push_begin(ctrl, depth, &position); + })) { ppu.test_state(); } @@ -465,7 +468,7 @@ error_code cellSyncQueuePush(ppu_thread& ppu, vm::ptr queue, vm:: // copy data from the buffer at the position std::memcpy(&queue->buffer[position * queue->size], buffer.get_ptr(), queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::push_end); + queue->ctrl.atomic_op<&CellSyncQueue::push_end>(); return CELL_OK; } @@ -488,7 +491,10 @@ error_code cellSyncQueueTryPush(vm::ptr queue, vm::cptr buf u32 position; - if (!queue->ctrl.atomic_op(&CellSyncQueue::try_push_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_push_begin(ctrl, depth, &position); + })) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -496,7 +502,7 @@ error_code cellSyncQueueTryPush(vm::ptr queue, vm::cptr buf // copy data from the buffer at the position std::memcpy(&queue->buffer[position * queue->size], buffer.get_ptr(), queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::push_end); + queue->ctrl.atomic_op<&CellSyncQueue::push_end>(); return CELL_OK; } @@ -519,7 +525,10 @@ error_code cellSyncQueuePop(ppu_thread& ppu, vm::ptr queue, vm::p u32 position; - while (!queue->ctrl.atomic_op(&CellSyncQueue::try_pop_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_pop_begin(ctrl, depth, &position); + })) { ppu.test_state(); } @@ -527,7 +536,7 @@ error_code cellSyncQueuePop(ppu_thread& ppu, vm::ptr queue, vm::p // copy data at the position to the buffer std::memcpy(buffer.get_ptr(), &queue->buffer[position % depth * queue->size], queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::pop_end); + queue->ctrl.atomic_op<&CellSyncQueue::pop_end>(); return CELL_OK; } @@ -550,7 +559,10 @@ error_code cellSyncQueueTryPop(vm::ptr queue, vm::ptr buffe u32 position; - if (!queue->ctrl.atomic_op(&CellSyncQueue::try_pop_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_pop_begin(ctrl, depth, &position); + })) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -558,7 +570,7 @@ error_code cellSyncQueueTryPop(vm::ptr queue, vm::ptr buffe // copy data at the position to the buffer std::memcpy(buffer.get_ptr(), &queue->buffer[position % depth * queue->size], queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::pop_end); + queue->ctrl.atomic_op<&CellSyncQueue::pop_end>(); return CELL_OK; } @@ -581,7 +593,10 @@ error_code cellSyncQueuePeek(ppu_thread& ppu, vm::ptr queue, vm:: u32 position; - while (!queue->ctrl.atomic_op(&CellSyncQueue::try_peek_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_peek_begin(ctrl, depth, &position); + })) { ppu.test_state(); } @@ -589,7 +604,7 @@ error_code cellSyncQueuePeek(ppu_thread& ppu, vm::ptr queue, vm:: // copy data at the position to the buffer std::memcpy(buffer.get_ptr(), &queue->buffer[position % depth * queue->size], queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::pop_end); + queue->ctrl.atomic_op<&CellSyncQueue::pop_end>(); return CELL_OK; } @@ -612,7 +627,10 @@ error_code cellSyncQueueTryPeek(vm::ptr queue, vm::ptr buff u32 position; - if (!queue->ctrl.atomic_op(&CellSyncQueue::try_peek_begin, depth, &position)) + while (!queue->ctrl.atomic_op([&](auto& ctrl) + { + return CellSyncQueue::try_peek_begin(ctrl, depth, &position); + })) { return not_an_error(CELL_SYNC_ERROR_BUSY); } @@ -620,7 +638,7 @@ error_code cellSyncQueueTryPeek(vm::ptr queue, vm::ptr buff // copy data at the position to the buffer std::memcpy(buffer.get_ptr(), &queue->buffer[position % depth * queue->size], queue->size); - queue->ctrl.atomic_op(&CellSyncQueue::pop_end); + queue->ctrl.atomic_op<&CellSyncQueue::pop_end>(); return CELL_OK; } @@ -660,12 +678,12 @@ error_code cellSyncQueueClear(ppu_thread& ppu, vm::ptr queue) const u32 depth = queue->check_depth(); - while (!queue->ctrl.atomic_op(&CellSyncQueue::try_clear_begin_1)) + while (!queue->ctrl.atomic_op<&CellSyncQueue::try_clear_begin_1>()) { ppu.test_state(); } - while (!queue->ctrl.atomic_op(&CellSyncQueue::try_clear_begin_2)) + while (!queue->ctrl.atomic_op<&CellSyncQueue::try_clear_begin_2>()) { ppu.test_state(); } diff --git a/rpcs3/Emu/Cell/Modules/cellSync.h b/rpcs3/Emu/Cell/Modules/cellSync.h index b457b8b3aa..a33c01a88d 100644 --- a/rpcs3/Emu/Cell/Modules/cellSync.h +++ b/rpcs3/Emu/Cell/Modules/cellSync.h @@ -2,8 +2,6 @@ #include "Utilities/BitField.h" - - // Return Codes enum CellSyncError : u32 { @@ -36,34 +34,34 @@ enum CellSyncError1 : u32 struct CellSyncMutex { - struct alignas(4) ctrl_t + struct alignas(4) Counter { be_t rel; be_t acq; - }; - atomic_t ctrl; - - static inline auto lock_begin(ctrl_t& ctrl) - { - return ctrl.acq++; - } - - static inline bool try_lock(ctrl_t& ctrl) - { - if (UNLIKELY(ctrl.rel != ctrl.acq)) + auto lock_begin() { - return false; + return acq++; } - ctrl.acq++; - return true; - } + bool try_lock() + { + if (UNLIKELY(rel != acq)) + { + return false; + } - static inline void unlock(ctrl_t& ctrl) - { - ctrl.rel++; - } + acq++; + return true; + } + + void unlock() + { + rel++; + } + }; + + atomic_t ctrl; }; CHECK_SIZE_ALIGN(CellSyncMutex, 4, 4); diff --git a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp index 573aa2faae..f3ff307498 100644 --- a/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_event_flag.cpp @@ -115,7 +115,10 @@ error_code sys_event_flag_wait(ppu_thread& ppu, u32 id, u64 bitptn, u32 mode, vm const auto flag = idm::get(id, [&](lv2_event_flag& flag) -> CellError { - if (flag.pattern.atomic_op(lv2_event_flag::check_pattern, bitptn, mode, &ppu.gpr[6])) + if (flag.pattern.atomic_op([&](u64& pat) + { + return lv2_event_flag::check_pattern(pat, bitptn, mode, &ppu.gpr[6]); + })) { // TODO: is it possible to return EPERM in this case? return {}; @@ -123,7 +126,10 @@ error_code sys_event_flag_wait(ppu_thread& ppu, u32 id, u64 bitptn, u32 mode, vm std::lock_guard lock(flag.mutex); - if (flag.pattern.atomic_op(lv2_event_flag::check_pattern, bitptn, mode, &ppu.gpr[6])) + if (flag.pattern.atomic_op([&](u64& pat) + { + return lv2_event_flag::check_pattern(pat, bitptn, mode, &ppu.gpr[6]); + })) { return {}; } @@ -208,7 +214,10 @@ error_code sys_event_flag_trywait(u32 id, u64 bitptn, u32 mode, vm::ptr res const auto flag = idm::check(id, [&](lv2_event_flag& flag) { - return flag.pattern.atomic_op(lv2_event_flag::check_pattern, bitptn, mode, &pattern); + return flag.pattern.atomic_op([&](u64& pat) + { + return lv2_event_flag::check_pattern(pat, bitptn, mode, &pattern); + }); }); if (!flag)