From 1b140c8e979d80fb690ef2ba97024b84f9b1ea7f Mon Sep 17 00:00:00 2001
From: Nekotekina <nekotekina@gmail.com>
Date: Thu, 18 Jul 2019 18:40:08 +0300
Subject: [PATCH] SPU Recompiler: optimize JIT memory consumption

Avoid rebuilding trampoline for every function at startup.
This should fix Out of Memory error in some cases.
---
 rpcs3/Emu/Cell/SPURecompiler.cpp | 42 +++++++++++++++++++++++---------
 rpcs3/Emu/Cell/SPURecompiler.h   |  6 +++++
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/rpcs3/Emu/Cell/SPURecompiler.cpp b/rpcs3/Emu/Cell/SPURecompiler.cpp
index 75bb3df296..eb454a6b61 100644
--- a/rpcs3/Emu/Cell/SPURecompiler.cpp
+++ b/rpcs3/Emu/Cell/SPURecompiler.cpp
@@ -463,6 +463,9 @@ void spu_cache::initialize()
 
 	if (compilers.size() && !func_list.empty())
 	{
+		LOG_NOTICE(SPU, "SPU Runtime: Building trampoline...");
+		spu_runtime::g_dispatcher[0] = compilers[0]->get_runtime().rebuild_ubertrampoline();
+
 		LOG_SUCCESS(SPU, "SPU Runtime: Built %u functions.", func_list.size());
 	}
 
@@ -564,6 +567,26 @@ bool spu_runtime::add(u64 last_reset_count, void* _where, spu_function_t compile
 	// Register function in PIC map
 	m_pic_map[{func.data() + _off, func.size() - _off}] = compiled;
 
+	if (fxm::check_unlocked<spu_cache>())
+	{
+		// Rebuild trampolines if necessary
+		if (const auto new_tr = rebuild_ubertrampoline())
+		{
+			g_dispatcher[0] = new_tr;
+		}
+		else
+		{
+			return false;
+		}
+	}
+
+	// Notify in lock destructor
+	lock.notify = true;
+	return true;
+}
+
+spu_function_t spu_runtime::rebuild_ubertrampoline()
+{
 	// Prepare sorted list
 	m_flat_list.clear();
 	m_flat_list.assign(m_pic_map.cbegin(), m_pic_map.cend());
@@ -586,18 +609,14 @@ bool spu_runtime::add(u64 last_reset_count, void* _where, spu_function_t compile
 	const auto _end = m_flat_list.end();
 	const u32 size0 = ::size32(m_flat_list);
 
-	if (size0 == 1)
-	{
-		g_dispatcher[0] = compiled;
-	}
-	else
+	if (size0 != 1)
 	{
 		// Allocate some writable executable memory
 		u8* const wxptr = jit_runtime::alloc(size0 * 22 + 14, 16);
 
 		if (!wxptr)
 		{
-			return false;
+			return nullptr;
 		}
 
 		// Raw assembly pointer
@@ -728,7 +747,7 @@ bool spu_runtime::add(u64 last_reset_count, void* _where, spu_function_t compile
 			if (w.level >= w.beg->first.size() || w.level >= it->first.size())
 			{
 				// If functions cannot be compared, assume smallest function
-				LOG_ERROR(SPU, "Trampoline simplified at 0x%x (level=%u)", func[0], w.level);
+				LOG_ERROR(SPU, "Trampoline simplified at ??? (level=%u)", w.level);
 				make_jump(0xe9, w.beg->second); // jmp rel32
 				continue;
 			}
@@ -760,7 +779,7 @@ bool spu_runtime::add(u64 last_reset_count, void* _where, spu_function_t compile
 
 			if (it == m_flat_list.end())
 			{
-				LOG_ERROR(SPU, "Trampoline simplified (II) at 0x%x (level=%u)", func[0], w.level);
+				LOG_ERROR(SPU, "Trampoline simplified (II) at ??? (level=%u)", w.level);
 				make_jump(0xe9, w.beg->second); // jmp rel32
 				continue;
 			}
@@ -871,12 +890,11 @@ bool spu_runtime::add(u64 last_reset_count, void* _where, spu_function_t compile
 		}
 
 		workload.clear();
-		g_dispatcher[0] = reinterpret_cast<spu_function_t>(reinterpret_cast<u64>(wxptr));
+		return reinterpret_cast<spu_function_t>(reinterpret_cast<u64>(wxptr));
 	}
 
-	// Notify in lock destructor
-	lock.notify = true;
-	return true;
+	// No trampoline required
+	return beg->second;
 }
 
 void* spu_runtime::find(u64 last_reset_count, const std::vector<u32>& func)
diff --git a/rpcs3/Emu/Cell/SPURecompiler.h b/rpcs3/Emu/Cell/SPURecompiler.h
index 5bccab7c7b..c5b653a575 100644
--- a/rpcs3/Emu/Cell/SPURecompiler.h
+++ b/rpcs3/Emu/Cell/SPURecompiler.h
@@ -84,6 +84,12 @@ public:
 	// Add compiled function and generate trampoline if necessary
 	bool add(u64 last_reset_count, void* where, spu_function_t compiled);
 
+private:
+	spu_function_t rebuild_ubertrampoline();
+
+	friend class spu_cache;
+public:
+
 	// Return opaque pointer for add()
 	void* find(u64 last_reset_count, const std::vector<u32>&);