From 9ba3e6d3f792d0d6c8b0257758f369ea17d48e36 Mon Sep 17 00:00:00 2001
From: Eladash <elad3356p@gmail.com>
Date: Sat, 22 May 2021 05:54:52 +0300
Subject: [PATCH] sys_mmapper: Fix IPC-enabled instances memory consumption

---
 rpcs3/Emu/Cell/lv2/sys_mmapper.cpp | 71 +++++++++---------------------
 rpcs3/Emu/Cell/lv2/sys_mmapper.h   |  2 +
 2 files changed, 24 insertions(+), 49 deletions(-)

diff --git a/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp b/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp
index d82b2ecc7c..b1f375f7f0 100644
--- a/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp
+++ b/rpcs3/Emu/Cell/lv2/sys_mmapper.cpp
@@ -42,6 +42,17 @@ lv2_memory::lv2_memory(u32 size, u32 align, u64 flags, u64 key, bool pshared, lv
 #endif
 }
 
+CellError lv2_memory::on_id_create()
+{
+	if (!exists && !ct->take(size))
+	{
+		return CELL_ENOMEM;
+	}
+
+	exists++;
+	return {};
+}
+
 template <bool exclusive = false>
 error_code create_lv2_shm(bool pshared, u64 ipc_key, u64 size, u32 align, u64 flags, lv2_memory_container* ct)
 {
@@ -168,14 +179,8 @@ error_code sys_mmapper_allocate_shared_memory(ppu_thread& ppu, u64 ipc_key, u64
 	// Get "default" memory container
 	auto& dct = g_fxo->get<lv2_memory_container>();
 
-	if (!dct.take(size))
-	{
-		return CELL_ENOMEM;
-	}
-
 	if (auto error = create_lv2_shm(ipc_key != SYS_MMAPPER_NO_SHM_KEY, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, &dct))
 	{
-		dct.used -= size;
 		return error;
 	}
 
@@ -222,30 +227,15 @@ error_code sys_mmapper_allocate_shared_memory_from_container(ppu_thread& ppu, u6
 	}
 	}
 
-	const auto ct = idm::get<lv2_memory_container>(cid, [&](lv2_memory_container& ct) -> CellError
-	{
-		// Try to get "physical memory"
-		if (!ct.take(size))
-		{
-			return CELL_ENOMEM;
-		}
-
-		return {};
-	});
+	const auto ct = idm::get<lv2_memory_container>(cid);
 
 	if (!ct)
 	{
 		return CELL_ESRCH;
 	}
 
-	if (ct.ret)
+	if (auto error = create_lv2_shm(ipc_key != SYS_MMAPPER_NO_SHM_KEY, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, ct.get()))
 	{
-		return ct.ret;
-	}
-
-	if (auto error = create_lv2_shm(ipc_key != SYS_MMAPPER_NO_SHM_KEY, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, ct.ptr.get()))
-	{
-		ct->used -= size;
 		return error;
 	}
 
@@ -342,14 +332,8 @@ error_code sys_mmapper_allocate_shared_memory_ext(ppu_thread& ppu, u64 ipc_key,
 	// Get "default" memory container
 	auto& dct = g_fxo->get<lv2_memory_container>();
 
-	if (!dct.take(size))
-	{
-		return CELL_ENOMEM;
-	}
-
 	if (auto error = create_lv2_shm<true>(true, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, &dct))
 	{
-		dct.used -= size;
 		return error;
 	}
 
@@ -438,30 +422,15 @@ error_code sys_mmapper_allocate_shared_memory_from_container_ext(ppu_thread& ppu
 		}
 	}
 
-	const auto ct = idm::get<lv2_memory_container>(cid, [&](lv2_memory_container& ct) -> CellError
-	{
-		// Try to get "physical memory"
-		if (!ct.take(size))
-		{
-			return CELL_ENOMEM;
-		}
-
-		return {};
-	});
+	const auto ct = idm::get<lv2_memory_container>(cid);
 
 	if (!ct)
 	{
 		return CELL_ESRCH;
 	}
 
-	if (ct.ret)
+	if (auto error = create_lv2_shm<true>(true, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, ct.get()))
 	{
-		return ct.ret;
-	}
-
-	if (auto error = create_lv2_shm<true>(true, ipc_key, size, flags & SYS_MEMORY_PAGE_SIZE_64K ? 0x10000 : 0x100000, flags, ct.ptr.get()))
-	{
-		ct->used -= size;
 		return error;
 	}
 
@@ -550,6 +519,13 @@ error_code sys_mmapper_free_shared_memory(ppu_thread& ppu, u32 mem_id)
 		}
 
 		lv2_obj::on_id_destroy(mem, mem.key, +mem.pshared);
+
+		if (!mem.exists)
+		{
+			// Return "physical memory" to the memory container
+			mem.ct->used -= mem.size;
+		}
+
 		return {};
 	});
 
@@ -563,9 +539,6 @@ error_code sys_mmapper_free_shared_memory(ppu_thread& ppu, u32 mem_id)
 		return mem.ret;
 	}
 
-	// Return "physical memory" to the memory container
-	mem->ct->used -= mem->size;
-
 	return CELL_OK;
 }
 
diff --git a/rpcs3/Emu/Cell/lv2/sys_mmapper.h b/rpcs3/Emu/Cell/lv2/sys_mmapper.h
index 4c02286e31..a34119f784 100644
--- a/rpcs3/Emu/Cell/lv2/sys_mmapper.h
+++ b/rpcs3/Emu/Cell/lv2/sys_mmapper.h
@@ -29,6 +29,8 @@ struct lv2_memory : lv2_obj
 	atomic_t<u32> counter{0};
 
 	lv2_memory(u32 size, u32 align, u64 flags, u64 key, bool pshared, lv2_memory_container* ct);
+
+	CellError on_id_create();
 };
 
 enum : u64