Kernel: Move Mutex into Locking/

Author: https://github.com/boricj Commit: f7f794e74a Pull-request: https://github.com/SerenityOS/serenity/pull/8851 Reviewed-by: https://github.com/awesomekling ✅ Reviewed-by: https://github.com/bgianfo
2025-04-27 14:58:46 +00:00 · 2021-07-18 09:10:27 +02:00 · 2021-07-18 09:10:27 +02:00 · f7f794e74a · 2024-07-18 07:21:00 +09:00
commit f7f794e74a
parent 479b07339c
27 changed files with 26 additions and 26 deletions
--- a/Kernel/Locking/Mutex.cpp
+++ b/Kernel/Locking/Mutex.cpp
@ -0,0 +1,432 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifdef LOCK_DEBUG
+#    include <AK/SourceLocation.h>
+#endif
+#include <Kernel/Debug.h>
+#include <Kernel/KSyms.h>
+#include <Kernel/Locking/Mutex.h>
+#include <Kernel/SpinLock.h>
+#include <Kernel/Thread.h>
+
+namespace Kernel {
+
+#if LOCK_DEBUG
+void Mutex::lock(Mode mode, const SourceLocation& location)
+#else
+void Mutex::lock(Mode mode)
+#endif
+{
+    // NOTE: This may be called from an interrupt handler (not an IRQ handler)
+    // and also from within critical sections!
+    VERIFY(!Processor::current().in_irq());
+    VERIFY(mode != Mode::Unlocked);
+    auto current_thread = Thread::current();
+
+    ScopedSpinLock lock(m_lock);
+    bool did_block = false;
+    Mode current_mode = m_mode;
+    switch (current_mode) {
+    case Mode::Unlocked: {
+        dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ ({}) {}: acquire {}, currently unlocked", this, m_name, mode_to_string(mode));
+        m_mode = mode;
+        VERIFY(!m_holder);
+        VERIFY(m_shared_holders.is_empty());
+        if (mode == Mode::Exclusive) {
+            m_holder = current_thread;
+        } else {
+            VERIFY(mode == Mode::Shared);
+            m_shared_holders.set(current_thread, 1);
+        }
+        VERIFY(m_times_locked == 0);
+        m_times_locked++;
+
+#if LOCK_DEBUG
+        if (current_thread) {
+            current_thread->holding_lock(*this, 1, location);
+        }
+#endif
+        return;
+    }
+    case Mode::Exclusive: {
+        VERIFY(m_holder);
+        if (m_holder != current_thread) {
+            block(*current_thread, mode, lock, 1);
+            did_block = true;
+            // If we blocked then m_mode should have been updated to what we requested
+            VERIFY(m_mode == mode);
+        }
+
+        if (m_mode == Mode::Exclusive) {
+            VERIFY(m_holder == current_thread);
+            VERIFY(m_shared_holders.is_empty());
+        } else if (did_block && mode == Mode::Shared) {
+            // Only if we blocked trying to acquire a shared lock the lock would have been converted
+            VERIFY(!m_holder);
+            VERIFY(!m_shared_holders.is_empty());
+            VERIFY(m_shared_holders.find(current_thread) != m_shared_holders.end());
+        }
+
+        if constexpr (LOCK_TRACE_DEBUG) {
+            if (mode == Mode::Exclusive)
+                dbgln("Mutex::lock @ {} ({}): acquire {}, currently exclusive, holding: {}", this, m_name, mode_to_string(mode), m_times_locked);
+            else
+                dbgln("Mutex::lock @ {} ({}): acquire exclusive (requested {}), currently exclusive, holding: {}", this, m_name, mode_to_string(mode), m_times_locked);
+        }
+
+        VERIFY(m_times_locked > 0);
+        if (!did_block) {
+            // if we didn't block we must still be an exclusive lock
+            VERIFY(m_mode == Mode::Exclusive);
+            m_times_locked++;
+        }
+
+#if LOCK_DEBUG
+        current_thread->holding_lock(*this, 1, location);
+#endif
+        return;
+    }
+    case Mode::Shared: {
+        VERIFY(!m_holder);
+        if (mode == Mode::Exclusive) {
+            if (m_shared_holders.size() == 1) {
+                auto it = m_shared_holders.begin();
+                if (it->key == current_thread) {
+                    it->value++;
+                    m_times_locked++;
+                    m_mode = Mode::Exclusive;
+                    m_holder = current_thread;
+                    m_shared_holders.clear();
+                    dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}): acquire {}, converted shared to exclusive lock, locks held {}", this, m_name, mode_to_string(mode), m_times_locked);
+                    return;
+                }
+            }
+
+            block(*current_thread, mode, lock, 1);
+            did_block = true;
+            VERIFY(m_mode == mode);
+        }
+
+        dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}): acquire {}, currently shared, locks held {}", this, m_name, mode_to_string(mode), m_times_locked);
+
+        VERIFY(m_times_locked > 0);
+        if (m_mode == Mode::Shared) {
+            VERIFY(!m_holder);
+            VERIFY(!did_block || m_shared_holders.contains(current_thread));
+        } else if (did_block) {
+            VERIFY(mode == Mode::Exclusive);
+            VERIFY(m_holder == current_thread);
+            VERIFY(m_shared_holders.is_empty());
+        }
+
+        if (!did_block) {
+            // if we didn't block we must still be a shared lock
+            VERIFY(m_mode == Mode::Shared);
+            m_times_locked++;
+            VERIFY(!m_shared_holders.is_empty());
+            auto it = m_shared_holders.find(current_thread);
+            if (it != m_shared_holders.end())
+                it->value++;
+            else
+                m_shared_holders.set(current_thread, 1);
+        }
+
+#if LOCK_DEBUG
+        current_thread->holding_lock(*this, 1, location);
+#endif
+        return;
+    }
+    default:
+        VERIFY_NOT_REACHED();
+    }
+}
+
+void Mutex::unlock()
+{
+    // NOTE: This may be called from an interrupt handler (not an IRQ handler)
+    // and also from within critical sections!
+    VERIFY(!Processor::current().in_irq());
+    auto current_thread = Thread::current();
+    ScopedSpinLock lock(m_lock);
+    Mode current_mode = m_mode;
+    if constexpr (LOCK_TRACE_DEBUG) {
+        if (current_mode == Mode::Shared)
+            dbgln("Mutex::unlock @ {} ({}): release {}, locks held: {}", this, m_name, mode_to_string(current_mode), m_times_locked);
+        else
+            dbgln("Mutex::unlock @ {} ({}): release {}, holding: {}", this, m_name, mode_to_string(current_mode), m_times_locked);
+    }
+
+    VERIFY(current_mode != Mode::Unlocked);
+
+    VERIFY(m_times_locked > 0);
+    m_times_locked--;
+
+    switch (current_mode) {
+    case Mode::Exclusive:
+        VERIFY(m_holder == current_thread);
+        VERIFY(m_shared_holders.is_empty());
+        if (m_times_locked == 0)
+            m_holder = nullptr;
+        break;
+    case Mode::Shared: {
+        VERIFY(!m_holder);
+        auto it = m_shared_holders.find(current_thread);
+        VERIFY(it != m_shared_holders.end());
+        if (it->value > 1) {
+            it->value--;
+        } else {
+            VERIFY(it->value > 0);
+            m_shared_holders.remove(it);
+        }
+        break;
+    }
+    default:
+        VERIFY_NOT_REACHED();
+    }
+
+#if LOCK_DEBUG
+    if (current_thread) {
+        current_thread->holding_lock(*this, -1, {});
+    }
+#endif
+
+    if (m_times_locked == 0) {
+        VERIFY(current_mode == Mode::Exclusive ? !m_holder : m_shared_holders.is_empty());
+
+        m_mode = Mode::Unlocked;
+        unblock_waiters(current_mode);
+    }
+}
+
+void Mutex::block(Thread& current_thread, Mode mode, ScopedSpinLock<SpinLock<u8>>& lock, u32 requested_locks)
+{
+    auto& blocked_thread_list = thread_list_for_mode(mode);
+    VERIFY(!blocked_thread_list.contains(current_thread));
+    blocked_thread_list.append(current_thread);
+
+    dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}) waiting...", this, m_name);
+    current_thread.block(*this, lock, requested_locks);
+    dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}) waited", this, m_name);
+
+    VERIFY(blocked_thread_list.contains(current_thread));
+    blocked_thread_list.remove(current_thread);
+}
+
+void Mutex::unblock_waiters(Mode previous_mode)
+{
+    VERIFY(m_times_locked == 0);
+    VERIFY(m_mode == Mode::Unlocked);
+
+    if (m_blocked_threads_list_exclusive.is_empty() && m_blocked_threads_list_shared.is_empty())
+        return;
+
+    auto unblock_shared = [&]() {
+        if (m_blocked_threads_list_shared.is_empty())
+            return false;
+        m_mode = Mode::Shared;
+        for (auto& thread : m_blocked_threads_list_shared) {
+            auto requested_locks = thread.unblock_from_lock(*this);
+            auto set_result = m_shared_holders.set(&thread, requested_locks);
+            VERIFY(set_result == AK::HashSetResult::InsertedNewEntry);
+            m_times_locked += requested_locks;
+        }
+        return true;
+    };
+    auto unblock_exclusive = [&]() {
+        if (auto* next_exclusive_thread = m_blocked_threads_list_exclusive.first()) {
+            m_mode = Mode::Exclusive;
+            m_times_locked = next_exclusive_thread->unblock_from_lock(*this);
+            m_holder = next_exclusive_thread;
+            return true;
+        }
+        return false;
+    };
+
+    if (previous_mode == Mode::Exclusive) {
+        if (!unblock_shared())
+            unblock_exclusive();
+    } else {
+        if (!unblock_exclusive())
+            unblock_shared();
+    }
+}
+
+auto Mutex::force_unlock_if_locked(u32& lock_count_to_restore) -> Mode
+{
+    // NOTE: This may be called from an interrupt handler (not an IRQ handler)
+    // and also from within critical sections!
+    VERIFY(!Processor::current().in_irq());
+    auto current_thread = Thread::current();
+    ScopedSpinLock lock(m_lock);
+    auto current_mode = m_mode;
+    switch (current_mode) {
+    case Mode::Exclusive: {
+        if (m_holder != current_thread) {
+            lock_count_to_restore = 0;
+            return Mode::Unlocked;
+        }
+
+        dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::force_unlock_if_locked @ {}: unlocking exclusive with lock count: {}", this, m_times_locked);
+#if LOCK_DEBUG
+        m_holder->holding_lock(*this, -(int)m_times_locked, {});
+#endif
+        m_holder = nullptr;
+        VERIFY(m_times_locked > 0);
+        lock_count_to_restore = m_times_locked;
+        m_times_locked = 0;
+        m_mode = Mode::Unlocked;
+        unblock_waiters(Mode::Exclusive);
+        break;
+    }
+    case Mode::Shared: {
+        VERIFY(!m_holder);
+        auto it = m_shared_holders.find(current_thread);
+        if (it == m_shared_holders.end()) {
+            lock_count_to_restore = 0;
+            return Mode::Unlocked;
+        }
+
+        dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::force_unlock_if_locked @ {}: unlocking exclusive with lock count: {}, total locks: {}",
+            this, it->value, m_times_locked);
+
+        VERIFY(it->value > 0);
+        lock_count_to_restore = it->value;
+        VERIFY(lock_count_to_restore > 0);
+#if LOCK_DEBUG
+        m_holder->holding_lock(*this, -(int)lock_count_to_restore, {});
+#endif
+        m_shared_holders.remove(it);
+        VERIFY(m_times_locked >= lock_count_to_restore);
+        m_times_locked -= lock_count_to_restore;
+        if (m_times_locked == 0) {
+            m_mode = Mode::Unlocked;
+            unblock_waiters(Mode::Shared);
+        }
+        break;
+    }
+    case Mode::Unlocked: {
+        lock_count_to_restore = 0;
+        break;
+    }
+    default:
+        VERIFY_NOT_REACHED();
+    }
+    return current_mode;
+}
+
+#if LOCK_DEBUG
+void Mutex::restore_lock(Mode mode, u32 lock_count, const SourceLocation& location)
+#else
+void Mutex::restore_lock(Mode mode, u32 lock_count)
+#endif
+{
+    VERIFY(mode != Mode::Unlocked);
+    VERIFY(lock_count > 0);
+    VERIFY(!Processor::current().in_irq());
+    auto current_thread = Thread::current();
+    bool did_block = false;
+    ScopedSpinLock lock(m_lock);
+    switch (mode) {
+    case Mode::Exclusive: {
+        auto previous_mode = m_mode;
+        bool need_to_block = false;
+        if (m_mode == Mode::Exclusive && m_holder != current_thread)
+            need_to_block = true;
+        else if (m_mode == Mode::Shared && (m_shared_holders.size() != 1 || !m_shared_holders.contains(current_thread)))
+            need_to_block = true;
+        if (need_to_block) {
+            block(*current_thread, Mode::Exclusive, lock, lock_count);
+            did_block = true;
+            // If we blocked then m_mode should have been updated to what we requested
+            VERIFY(m_mode == Mode::Exclusive);
+        }
+
+        dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::restore_lock @ {}: restoring {} with lock count {}, was {}", this, mode_to_string(mode), lock_count, mode_to_string(previous_mode));
+
+        VERIFY(m_mode != Mode::Shared);
+        VERIFY(m_shared_holders.is_empty());
+        if (did_block) {
+            VERIFY(m_times_locked > 0);
+            VERIFY(m_holder == current_thread);
+        } else {
+            if (m_mode == Mode::Unlocked) {
+                m_mode = Mode::Exclusive;
+                VERIFY(m_times_locked == 0);
+                m_times_locked = lock_count;
+                VERIFY(!m_holder);
+                m_holder = current_thread;
+            } else if (m_mode == Mode::Shared) {
+                // Upgrade the shared lock to an exclusive lock
+                VERIFY(!m_holder);
+                VERIFY(m_shared_holders.size() == 1);
+                VERIFY(m_shared_holders.contains(current_thread));
+                m_mode = Mode::Exclusive;
+                m_holder = current_thread;
+                m_shared_holders.clear();
+            } else {
+                VERIFY(m_mode == Mode::Exclusive);
+                VERIFY(m_holder == current_thread);
+                VERIFY(m_times_locked > 0);
+                m_times_locked += lock_count;
+            }
+        }
+
+#if LOCK_DEBUG
+        m_holder->holding_lock(*this, (int)lock_count, location);
+#endif
+        return;
+    }
+    case Mode::Shared: {
+        auto previous_mode = m_mode;
+        if (m_mode == Mode::Exclusive && m_holder != current_thread) {
+            block(*current_thread, Mode::Shared, lock, lock_count);
+            did_block = true;
+            // If we blocked then m_mode should have been updated to what we requested
+            VERIFY(m_mode == Mode::Shared);
+        }
+
+        dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::restore_lock @ {}: restoring {} with lock count {}, was {}",
+            this, mode_to_string(mode), lock_count, mode_to_string(previous_mode));
+
+        VERIFY(!m_holder);
+        if (did_block) {
+            VERIFY(m_times_locked > 0);
+            VERIFY(m_shared_holders.contains(current_thread));
+        } else {
+            if (m_mode == Mode::Unlocked) {
+                m_mode = Mode::Shared;
+                m_times_locked += lock_count;
+                auto set_result = m_shared_holders.set(current_thread, lock_count);
+                // There may be other shared lock holders already, but we should not have an entry yet
+                VERIFY(set_result == AK::HashSetResult::InsertedNewEntry);
+            } else if (m_mode == Mode::Shared) {
+                m_times_locked += lock_count;
+                if (auto it = m_shared_holders.find(current_thread); it != m_shared_holders.end()) {
+                    it->value += lock_count;
+                } else {
+                    auto set_result = m_shared_holders.set(current_thread, lock_count);
+                    // There may be other shared lock holders already, but we should not have an entry yet
+                    VERIFY(set_result == AK::HashSetResult::InsertedNewEntry);
+                }
+            } else {
+                VERIFY(m_mode == Mode::Exclusive);
+                VERIFY(m_holder == current_thread);
+                m_times_locked += lock_count;
+            }
+        }
+
+#if LOCK_DEBUG
+        m_holder->holding_lock(*this, (int)lock_count, location);
+#endif
+        return;
+    }
+    default:
+        VERIFY_NOT_REACHED();
+    }
+}
+
+}