diff --git a/src/core/hle/service/nvnflinger/buffer_queue_consumer.cpp b/src/core/hle/service/nvnflinger/buffer_queue_consumer.cpp
index 51291539d9..e90d95b8fa 100644
--- a/src/core/hle/service/nvnflinger/buffer_queue_consumer.cpp
+++ b/src/core/hle/service/nvnflinger/buffer_queue_consumer.cpp
@@ -5,7 +5,6 @@
 // https://cs.android.com/android/platform/superproject/+/android-5.1.1_r38:frameworks/native/libs/gui/BufferQueueConsumer.cpp
 
 #include "common/logging/log.h"
-#include "core/hle/service/nvdrv/core/nvmap.h"
 #include "core/hle/service/nvnflinger/buffer_item.h"
 #include "core/hle/service/nvnflinger/buffer_queue_consumer.h"
 #include "core/hle/service/nvnflinger/buffer_queue_core.h"
@@ -14,9 +13,8 @@
 
 namespace Service::android {
 
-BufferQueueConsumer::BufferQueueConsumer(std::shared_ptr<BufferQueueCore> core_,
-                                         Service::Nvidia::NvCore::NvMap& nvmap_)
-    : core{std::move(core_)}, slots{core->slots}, nvmap(nvmap_) {}
+BufferQueueConsumer::BufferQueueConsumer(std::shared_ptr<BufferQueueCore> core_)
+    : core{std::move(core_)}, slots{core->slots} {}
 
 BufferQueueConsumer::~BufferQueueConsumer() = default;
 
@@ -135,8 +133,7 @@ Status BufferQueueConsumer::ReleaseBuffer(s32 slot, u64 frame_number, const Fenc
         }
 
         slots[slot].buffer_state = BufferState::Free;
-
-        nvmap.FreeHandle(slots[slot].graphic_buffer->BufferId(), true);
+        slots[slot].map_handle.reset();
 
         listener = core->connected_producer_listener;
 
diff --git a/src/core/hle/service/nvnflinger/buffer_queue_consumer.h b/src/core/hle/service/nvnflinger/buffer_queue_consumer.h
index 50ed0bb5fb..d32a00c40b 100644
--- a/src/core/hle/service/nvnflinger/buffer_queue_consumer.h
+++ b/src/core/hle/service/nvnflinger/buffer_queue_consumer.h
@@ -25,8 +25,7 @@ class IConsumerListener;
 
 class BufferQueueConsumer final {
 public:
-    explicit BufferQueueConsumer(std::shared_ptr<BufferQueueCore> core_,
-                                 Service::Nvidia::NvCore::NvMap& nvmap_);
+    explicit BufferQueueConsumer(std::shared_ptr<BufferQueueCore> core_);
     ~BufferQueueConsumer();
 
     Status AcquireBuffer(BufferItem* out_buffer, std::chrono::nanoseconds expected_present);
@@ -37,7 +36,6 @@ public:
 private:
     std::shared_ptr<BufferQueueCore> core;
     BufferQueueDefs::SlotsType& slots;
-    Service::Nvidia::NvCore::NvMap& nvmap;
 };
 
 } // namespace Service::android
diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
index dc6917d5db..5d6388d534 100644
--- a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
+++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp
@@ -24,6 +24,33 @@
 
 namespace Service::android {
 
+MapHandle::MapHandle() : m_nvmap(), m_nvmap_id() {}
+MapHandle::~MapHandle() {
+    this->reset();
+}
+
+MapHandle& MapHandle::operator=(const MapHandle& rhs) {
+    this->reset();
+    m_nvmap = rhs.m_nvmap;
+    m_nvmap_id = rhs.m_nvmap_id;
+    return *this;
+}
+
+void MapHandle::emplace(Service::Nvidia::NvCore::NvMap& nvmap, u32 nvmap_id) {
+    this->reset();
+
+    m_nvmap = std::addressof(nvmap);
+    m_nvmap_id = nvmap_id;
+    m_nvmap->DuplicateHandle(m_nvmap_id, true);
+}
+
+void MapHandle::reset() {
+    if (m_nvmap) {
+        m_nvmap->FreeHandle(m_nvmap_id, true);
+        m_nvmap = nullptr;
+    }
+}
+
 BufferQueueProducer::BufferQueueProducer(Service::KernelHelpers::ServiceContext& service_context_,
                                          std::shared_ptr<BufferQueueCore> buffer_queue_core_,
                                          Service::Nvidia::NvCore::NvMap& nvmap_)
@@ -532,7 +559,7 @@ Status BufferQueueProducer::QueueBuffer(s32 slot, const QueueBufferInput& input,
         item.is_droppable = core->dequeue_buffer_cannot_block || async;
         item.swap_interval = swap_interval;
 
-        nvmap.DuplicateHandle(item.graphic_buffer->BufferId(), true);
+        slots[slot].map_handle.emplace(nvmap, item.graphic_buffer->BufferId());
 
         sticky_transform = sticky_transform_;
 
@@ -745,8 +772,8 @@ Status BufferQueueProducer::Disconnect(NativeWindowApi api) {
 
         // HACK: We are not Android. Remove handle for items in queue, and clear queue.
         // Allows synchronous destruction of nvmap handles.
-        for (auto& item : core->queue) {
-            nvmap.FreeHandle(item.graphic_buffer->BufferId(), true);
+        for (auto& slot : slots) {
+            slot.map_handle.reset();
         }
         core->queue.clear();
 
diff --git a/src/core/hle/service/nvnflinger/buffer_slot.h b/src/core/hle/service/nvnflinger/buffer_slot.h
index d8c9dec3b2..6c55e27bee 100644
--- a/src/core/hle/service/nvnflinger/buffer_slot.h
+++ b/src/core/hle/service/nvnflinger/buffer_slot.h
@@ -11,6 +11,10 @@
 #include "common/common_types.h"
 #include "core/hle/service/nvnflinger/ui/fence.h"
 
+namespace Service::Nvidia::NvCore {
+class NvMap;
+}
+
 namespace Service::android {
 
 struct GraphicBuffer;
@@ -22,10 +26,27 @@ enum class BufferState : u32 {
     Acquired = 3,
 };
 
+class MapHandle final {
+public:
+    MapHandle();
+    ~MapHandle();
+
+    MapHandle(const MapHandle& rhs) = delete;
+    MapHandle& operator=(const MapHandle& rhs);
+
+    void emplace(Service::Nvidia::NvCore::NvMap& nvmap, u32 nvmap_id);
+    void reset();
+
+private:
+    Service::Nvidia::NvCore::NvMap* m_nvmap;
+    u32 m_nvmap_id;
+};
+
 struct BufferSlot final {
-    constexpr BufferSlot() = default;
+    BufferSlot() = default;
 
     std::shared_ptr<GraphicBuffer> graphic_buffer;
+    MapHandle map_handle;
     BufferState buffer_state{BufferState::Free};
     bool request_buffer_called{};
     u64 frame_number{};
diff --git a/src/core/hle/service/vi/display/vi_display.cpp b/src/core/hle/service/vi/display/vi_display.cpp
index f0b5eff8af..d30f49877a 100644
--- a/src/core/hle/service/vi/display/vi_display.cpp
+++ b/src/core/hle/service/vi/display/vi_display.cpp
@@ -35,7 +35,7 @@ static BufferQueue CreateBufferQueue(KernelHelpers::ServiceContext& service_cont
     return {
         buffer_queue_core,
         std::make_unique<android::BufferQueueProducer>(service_context, buffer_queue_core, nvmap),
-        std::make_unique<android::BufferQueueConsumer>(buffer_queue_core, nvmap)};
+        std::make_unique<android::BufferQueueConsumer>(buffer_queue_core)};
 }
 
 Display::Display(u64 id, std::string name_,