diff --git a/include/audio/hle_core.hpp b/include/audio/hle_core.hpp
index f04ed6c2..04c20a62 100644
--- a/include/audio/hle_core.hpp
+++ b/include/audio/hle_core.hpp
@@ -146,7 +146,7 @@ namespace Audio {
 
 		std::array<std::vector<u8>, pipeCount> pipeData;      // The data of each pipe
 		std::array<Source, Audio::HLE::sourceCount> sources;  // DSP voices
-		Audio::HLE::DspMemory dspRam;
+		Audio::HLE::DspMemory& dspRam;
 
 		Audio::DSPMixer mixer;
 		std::unique_ptr<Audio::AAC::Decoder> aacDecoder;
diff --git a/include/audio/teakra_core.hpp b/include/audio/teakra_core.hpp
index 46cf7c00..66819b44 100644
--- a/include/audio/teakra_core.hpp
+++ b/include/audio/teakra_core.hpp
@@ -20,6 +20,7 @@ namespace Audio {
 
 		// Get a pointer to a data memory address
 		u8* getDataPointer(u32 address) { return getDspMemory() + Memory::DSP_DATA_MEMORY_OFFSET + address; }
+		Teakra::UserConfig getTeakraConfig();
 
 		enum class PipeDirection {
 			DSPtoCPU = 0,
@@ -89,7 +90,7 @@ namespace Audio {
 		}
 
 		void setAudioEnabled(bool enable) override;
-		u8* getDspMemory() override { return teakra.GetDspMemory().data(); }
+		u8* getDspMemory() override { return teakra.GetDspMemory(); }
 		void* getRegisters() override;
 		DSPCore::Type getType() override { return DSPCore::Type::Teakra; }
 		u16 readProgramWord(u32 address) override { return teakra.ProgramRead(address); }
diff --git a/src/core/audio/dsp_core.cpp b/src/core/audio/dsp_core.cpp
index c793fcf8..b632cfa4 100644
--- a/src/core/audio/dsp_core.cpp
+++ b/src/core/audio/dsp_core.cpp
@@ -22,7 +22,6 @@ std::unique_ptr<Audio::DSPCore> Audio::makeDSPCore(EmulatorConfig& config, Memor
 			break;
 	}
 
-	mem.setDSPMem(core->getDspMemory());
 	return core;
 }
 
diff --git a/src/core/audio/hle_core.cpp b/src/core/audio/hle_core.cpp
index 059318c8..092f94a8 100644
--- a/src/core/audio/hle_core.cpp
+++ b/src/core/audio/hle_core.cpp
@@ -23,7 +23,7 @@ namespace Audio {
 	}
 
 	HLE_DSP::HLE_DSP(Memory& mem, Scheduler& scheduler, DSPService& dspService, EmulatorConfig& config)
-		: DSPCore(mem, scheduler, dspService, config) {
+		: DSPCore(mem, scheduler, dspService, config), dspRam(*(Audio::HLE::DspMemory*)mem.getDSPMem()) {
 		// Set up source indices
 		for (int i = 0; i < sources.size(); i++) {
 			sources[i].index = i;
diff --git a/src/core/audio/teakra_core.cpp b/src/core/audio/teakra_core.cpp
index 05d39487..cc5d6170 100644
--- a/src/core/audio/teakra_core.cpp
+++ b/src/core/audio/teakra_core.cpp
@@ -11,7 +11,7 @@
 using namespace Audio;
 
 TeakraDSP::TeakraDSP(Memory& mem, Scheduler& scheduler, DSPService& dspService, EmulatorConfig& config)
-	: DSPCore(mem, scheduler, dspService, config), pipeBaseAddr(0), running(false) {
+	: DSPCore(mem, scheduler, dspService, config), pipeBaseAddr(0), teakra(getTeakraConfig()), running(false) {
 	// Set up callbacks for Teakra
 	Teakra::AHBMCallback ahbm;
 
@@ -82,6 +82,13 @@ TeakraDSP::TeakraDSP(Memory& mem, Scheduler& scheduler, DSPService& dspService,
 	teakra.SetSemaphoreHandler([processPipeEvent]() { processPipeEvent(false); });
 }
 
+Teakra::UserConfig TeakraDSP::getTeakraConfig() {
+	Teakra::UserConfig config;
+	config.dsp_memory = mem.getDSPMem();
+
+	return config;
+}
+
 void TeakraDSP::reset() {
 	teakra.Reset();
 	running = false;
@@ -237,7 +244,7 @@ void TeakraDSP::loadComponent(std::vector<u8>& data, u32 programMask, u32 dataMa
 	teakra.Reset();
 	running = true;
 
-	u8* dspCode = teakra.GetDspMemory().data();
+	u8* dspCode = teakra.GetDspMemory();
 	u8* dspData = dspCode + 0x40000;
 
 	Dsp1 dsp1;
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 650c8d0e..7e429a1f 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -24,7 +24,7 @@ Memory::Memory(KFcram& fcramManager, const EmulatorConfig& config) : fcramManage
 	paddrTable.resize(totalPageCount, 0);
 
 	fcram = arena->BackingBasePointer() + FASTMEM_FCRAM_OFFSET;
-	// arenaDSPRam = arena->BackingBasePointer() + FASTMEM_DSP_RAM_OFFSET;
+	dspRam = arena->BackingBasePointer() + FASTMEM_DSP_RAM_OFFSET;
 	useFastmem = fastmemEnabled && arena->VirtualBasePointer() != nullptr;
 }
 
@@ -395,6 +395,10 @@ void Memory::mapPhysicalMemory(u32 vaddr, u32 paddr, s32 pages, bool r, bool w,
 		}
 	} else if (paddr >= VirtualAddrs::DSPMemStart && paddr < VirtualAddrs::DSPMemStart + DSP_RAM_SIZE) {
 		hostPtr = dspRam + (paddr - VirtualAddrs::DSPMemStart);
+
+		if (useFastmem) {
+			addFastmemView(vaddr, FASTMEM_DSP_RAM_OFFSET + paddr - VirtualAddrs::DSPMemStart, usize(pages) * pageSize, w);
+		}
 	}
 
 	for (int i = 0; i < pages; i++) {