diff --git a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp
index 54c9005d2a..0c2582bdab 100644
--- a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp
+++ b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp
@@ -151,6 +151,9 @@ CEXISlippi::CEXISlippi(Core::System& system, const std::string current_file_name
   directCodes = std::make_unique<SlippiDirectCodes>("direct-codes.json");
   teamsCodes = std::make_unique<SlippiDirectCodes>("teams-codes.json");
 
+  // initialize the spectate server so we can connect without starting a game
+  SlippiSpectateServer::getInstance();
+
   generator = std::default_random_engine(Common::Timer::NowMs());
 
   // Loggers will check 5 bytes, make sure we own that memory
diff --git a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.h b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.h
index 5d0a0cb96e..b7089cd5d4 100644
--- a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.h
+++ b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.h
@@ -262,7 +262,6 @@ private:
 
   std::vector<u8> m_read_queue;
   std::unique_ptr<Slippi::SlippiGame> m_current_game = nullptr;
-  SlippiSpectateServer* m_slippiserver = nullptr;
   SlippiMatchmaking::MatchSearchSettings last_search;
   SlippiMatchmaking::MatchmakeResult recent_mm_result;
 
diff --git a/Source/Core/Core/Slippi/SlippiSpectate.cpp b/Source/Core/Core/Slippi/SlippiSpectate.cpp
index 0288337246..07229ce076 100644
--- a/Source/Core/Core/Slippi/SlippiSpectate.cpp
+++ b/Source/Core/Core/Slippi/SlippiSpectate.cpp
@@ -280,7 +280,7 @@ void SlippiSpectateServer::SlippicommSocketThread(void)
 
   ENetAddress server_address = {0};
   server_address.host = ENET_HOST_ANY;
-  server_address.port = Config::Get(Config::SLIPPI_ENABLE_SPECTATOR);
+  server_address.port = Config::Get(Config::SLIPPI_SPECTATOR_LOCAL_PORT);
 
   // Create the spectator server
   // This call can fail if the system is already listening on the specified port