diff --git a/rpcs3/Emu/Cell/lv2/sys_memory.cpp b/rpcs3/Emu/Cell/lv2/sys_memory.cpp
index d8c5d8e3dc..6f2771f032 100644
--- a/rpcs3/Emu/Cell/lv2/sys_memory.cpp
+++ b/rpcs3/Emu/Cell/lv2/sys_memory.cpp
@@ -4,20 +4,17 @@
 #include "Utilities/VirtualMemory.h"
 #include "Emu/Memory/vm_locking.h"
 #include "Emu/IdManager.h"
+#include <shared_mutex>
 
 LOG_CHANNEL(sys_memory);
 
 //
 static shared_mutex s_memstats_mtx;
 
-lv2_memory_alloca::lv2_memory_alloca(u32 size, u32 align, u64 flags, const std::shared_ptr<lv2_memory_container>& ct)
-	: size(size)
-	, align(align)
-	, flags(flags)
-	, ct(ct)
-	, shm(std::make_shared<utils::shm>(size))
+struct sys_memory_address_table
 {
-}
+	atomic_t<lv2_memory_container*> addrs[65536]{};
+};
 
 // Todo: fix order of error checks
 
@@ -27,6 +24,11 @@ error_code sys_memory_allocate(u32 size, u64 flags, vm::ptr<u32> alloc_addr)
 
 	sys_memory.warning("sys_memory_allocate(size=0x%x, flags=0x%llx, alloc_addr=*0x%x)", size, flags, alloc_addr);
 
+	if (!size)
+	{
+		return {CELL_EALIGN, size};
+	}
+
 	// Check allocation size
 	const u32 align =
 		flags == SYS_MEMORY_PAGE_SIZE_1M ? 0x100000 :
@@ -56,6 +58,8 @@ error_code sys_memory_allocate(u32 size, u64 flags, vm::ptr<u32> alloc_addr)
 	{
 		if (u32 addr = area->alloc(size, align))
 		{
+			verify(HERE), !g_fxo->get<sys_memory_address_table>()->addrs[addr >> 16].exchange(dct);
+
 			if (alloc_addr)
 			{
 				*alloc_addr = addr;
@@ -78,6 +82,11 @@ error_code sys_memory_allocate_from_container(u32 size, u32 cid, u64 flags, vm::
 
 	sys_memory.warning("sys_memory_allocate_from_container(size=0x%x, cid=0x%x, flags=0x%llx, alloc_addr=*0x%x)", size, cid, flags, alloc_addr);
 
+	if (!size)
+	{
+		return {CELL_EALIGN, size};
+	}
+
 	// Check allocation size
 	const u32 align =
 		flags == SYS_MEMORY_PAGE_SIZE_1M ? 0x100000 :
@@ -115,13 +124,12 @@ error_code sys_memory_allocate_from_container(u32 size, u32 cid, u64 flags, vm::
 		return ct.ret;
 	}
 
-	// Create phantom memory object
-	const auto mem = idm::make_ptr<lv2_memory_alloca>(size, align, flags, ct.ptr);
-
 	if (const auto area = vm::reserve_map(align == 0x10000 ? vm::user64k : vm::user1m, 0, ::align(size, 0x10000000), 0x401))
 	{
-		if (u32 addr = area->alloc(size, mem->align, &mem->shm))
+		if (u32 addr = area->alloc(size))
 		{
+			verify(HERE), !g_fxo->get<sys_memory_address_table>()->addrs[addr >> 16].exchange(ct.ptr.get());
+
 			if (alloc_addr)
 			{
 				*alloc_addr = addr;
@@ -134,7 +142,6 @@ error_code sys_memory_allocate_from_container(u32 size, u32 cid, u64 flags, vm::
 		}
 	}
 
-	idm::remove<lv2_memory_alloca>(idm::last_id());
 	ct->used -= size;
 	return CELL_ENOMEM;
 }
@@ -145,57 +152,15 @@ error_code sys_memory_free(u32 addr)
 
 	sys_memory.warning("sys_memory_free(addr=0x%x)", addr);
 
-	const auto area = vm::get(vm::any, addr);
+	const auto ct = addr % 0x10000 ? nullptr : g_fxo->get<sys_memory_address_table>()->addrs[addr >> 16].exchange(nullptr);
 
-	if (!area || (area->flags & 3) != 1)
+	if (!ct)
 	{
 		return {CELL_EINVAL, addr};
 	}
 
-	const auto shm = area->get(addr);
-
-	if (!shm.second)
-	{
-		return {CELL_EINVAL, addr};
-	}
-
-	// Retrieve phantom memory object
-	const auto mem = idm::select<lv2_memory_alloca>([&](u32 id, lv2_memory_alloca& mem) -> u32
-	{
-		if (mem.shm.get() == shm.second.get())
-		{
-			return id;
-		}
-
-		return 0;
-	});
-
-	if (!mem)
-	{
-		// Deallocate memory (simple)
-		if (!area->dealloc(addr))
-		{
-			return {CELL_EINVAL, addr};
-		}
-
-		// Return "physical memory" to the default container
-		g_fxo->get<lv2_memory_container>()->used -= shm.second->size();
-
-		return CELL_OK;
-	}
-
-	// Deallocate memory
-	if (!area->dealloc(addr, &shm.second))
-	{
-		return {CELL_EINVAL, addr};
-	}
-
-	// Return "physical memory"
-	mem->ct->used -= mem->size;
-
-	// Remove phantom memory object
-	verify(HERE), idm::remove<lv2_memory_alloca>(mem.ret);
-
+	const auto size = verify(HERE, vm::dealloc(addr));
+	std::shared_lock{id_manager::g_mutex}, ct->used -= size;
 	return CELL_OK;
 }
 
diff --git a/rpcs3/Emu/Cell/lv2/sys_memory.h b/rpcs3/Emu/Cell/lv2/sys_memory.h
index 1749e3bb63..dcf4c66f1a 100644
--- a/rpcs3/Emu/Cell/lv2/sys_memory.h
+++ b/rpcs3/Emu/Cell/lv2/sys_memory.h
@@ -76,21 +76,6 @@ struct lv2_memory_container
 	}
 };
 
-struct lv2_memory_alloca
-{
-	static const u32 id_base = 0x1;
-	static const u32 id_step = 0x1;
-	static const u32 id_count = 0x2000;
-
-	const u32 size; // Memory size
-	const u32 align; // Alignment required
-	const u64 flags;
-	const std::shared_ptr<lv2_memory_container> ct;
-	const std::shared_ptr<utils::shm> shm;
-
-	lv2_memory_alloca(u32 size, u32 align, u64 flags, const std::shared_ptr<lv2_memory_container>& ct);
-};
-
 struct sys_memory_user_memory_stat_t
 {
 	be_t<u32> a; // 0x0