mirrors/dolphin
0
0
Fork 0
mirror of https://github.com/dolphin-emu/dolphin.git synced 2025-08-22 10:19:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add desync replay reporting

Browse source 
64feb0f
This commit is contained in:
Nikhil Narayana 2023-07-15 13:32:54 -07:00 • committed by David Liu
commit 0a66086f1e
13 changed files with 232 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Source/Core/Core/Core.cpp
View file

@ -532,7 +532,8 @@ static void EmuThread(std::unique_ptr<BootParameters> boot, WindowSystemInfo wsi
AudioCommon::InitSoundStream(system);
Common::ScopeGuard audio_guard([&system] { AudioCommon::ShutdownSoundStream(system); });
HW::Init(NetPlay::IsNetPlayRunning() ? &(boot_session_data.GetNetplaySettings()->sram) : nullptr);
std::string current_file_name = std::get<BootParameters::Disc>(boot->parameters).path;
HW::Init(NetPlay::IsNetPlayRunning() ? &(boot_session_data.GetNetplaySettings()->sram) : nullptr, current_file_name);
Common::ScopeGuard hw_guard{[&system] {
// We must set up this flag before executing HW::Shutdown()

4
Source/Core/Core/HW/EXI/EXI.cpp
View file

@ -96,7 +96,7 @@ u8 SlotToEXIDevice(Slot slot)
}
}
void ExpansionInterfaceManager::Init(const Sram* override_sram)
void ExpansionInterfaceManager::Init(const Sram* override_sram, const std::string current_file_name)
{
auto& sram = m_system.GetSRAM();
if (override_sram)
@ -130,7 +130,7 @@ void ExpansionInterfaceManager::Init(const Sram* override_sram)
Memcard::HeaderData header_data;
Memcard::InitializeHeaderData(&header_data, flash_id, size_mbits, shift_jis, rtc_bias,
sram_language, format_time + i);
m_channels[i] = std::make_unique<CEXIChannel>(m_system, i, header_data);
m_channels[i] = std::make_unique<CEXIChannel>(m_system, i, header_data, current_file_name);
}
}

2
Source/Core/Core/HW/EXI/EXI.h
View file

@ -69,7 +69,7 @@ public:
ExpansionInterfaceManager& operator=(ExpansionInterfaceManager&&) = delete;
~ExpansionInterfaceManager();
void Init(const Sram* override_sram);
void Init(const Sram* override_sram, const std::string current_file_name);
void Shutdown();
void DoState(PointerWrap& p);
void PauseAndLock(bool doLock, bool unpauseOnUnlock);

11
Source/Core/Core/HW/EXI/EXI_Channel.cpp
View file

@ -26,8 +26,9 @@ enum
};
CEXIChannel::CEXIChannel(Core::System& system, u32 channel_id,
const Memcard::HeaderData& memcard_header_data)
: m_system(system), m_channel_id(channel_id), m_memcard_header_data(memcard_header_data)
const Memcard::HeaderData& memcard_header_data,
const std::string current_file_name)
: m_system(system), m_channel_id(channel_id), m_memcard_header_data(memcard_header_data), m_current_file_name(current_file_name)
{
if (m_channel_id == 0 || m_channel_id == 1)
m_status.EXTINT = 1;
@ -35,7 +36,7 @@ CEXIChannel::CEXIChannel(Core::System& system, u32 channel_id,
m_status.CHIP_SELECT = 1;
for (auto& device : m_devices)
device = EXIDevice_Create(system, EXIDeviceType::None, m_channel_id, m_memcard_header_data);
device = EXIDevice_Create(system, EXIDeviceType::None, m_channel_id, m_memcard_header_data, m_current_file_name);
}
CEXIChannel::~CEXIChannel()
@ -171,7 +172,7 @@ void CEXIChannel::RemoveDevices()
void CEXIChannel::AddDevice(const EXIDeviceType device_type, const int device_num)
{
AddDevice(EXIDevice_Create(m_system, device_type, m_channel_id, m_memcard_header_data),
AddDevice(EXIDevice_Create(m_system, device_type, m_channel_id, m_memcard_header_data, m_current_file_name),
device_num);
}
@ -257,7 +258,7 @@ void CEXIChannel::DoState(PointerWrap& p)
else
{
std::unique_ptr<IEXIDevice> save_device =
EXIDevice_Create(m_system, type, m_channel_id, m_memcard_header_data);
EXIDevice_Create(m_system, type, m_channel_id, m_memcard_header_data, m_current_file_name);
save_device->DoState(p);
AddDevice(std::move(save_device), device_index, false);
}

6
Source/Core/Core/HW/EXI/EXI_Channel.h
View file

@ -26,7 +26,8 @@ class CEXIChannel
{
public:
explicit CEXIChannel(Core::System& system, u32 channel_id,
const Memcard::HeaderData& memcard_header_data);
const Memcard::HeaderData& memcard_header_data,
std::string current_file_name);
~CEXIChannel();
// get device
@ -119,6 +120,9 @@ private:
// it, as this class creates the CEXIMemoryCard instances.
Memcard::HeaderData m_memcard_header_data;
// used by SlippiGameReporter for calculating the md5
std::string m_current_file_name;
// Devices
enum
{

5
Source/Core/Core/HW/EXI/EXI_Device.cpp
View file

@ -106,7 +106,8 @@ void IEXIDevice::TransferByte(u8& byte)
// F A C T O R Y
std::unique_ptr<IEXIDevice> EXIDevice_Create(Core::System& system, const EXIDeviceType device_type,
const int channel_num,
const Memcard::HeaderData& memcard_header_data)
const Memcard::HeaderData& memcard_header_data,
const std::string current_file_name)
{
std::unique_ptr<IEXIDevice> result;
// XXX This computation isn't necessarily right (it holds for A/B, but not SP1)
@ -165,7 +166,7 @@ std::unique_ptr<IEXIDevice> EXIDevice_Create(Core::System& system, const EXIDevi
break;
case EXIDeviceType::Slippi:
result = std::make_unique<CEXISlippi>(system);
result = std::make_unique<CEXISlippi>(system, current_file_name);
break;
case EXIDeviceType::AMBaseboard:

3
Source/Core/Core/HW/EXI/EXI_Device.h
View file

@ -85,7 +85,8 @@ private:
std::unique_ptr<IEXIDevice> EXIDevice_Create(Core::System& system, EXIDeviceType device_type,
int channel_num,
const Memcard::HeaderData& memcard_header_data);
const Memcard::HeaderData& memcard_header_data,
const std::string current_file_name);
} // namespace ExpansionInterface
template <>

8
Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp
View file

@ -116,7 +116,7 @@ std::string ConvertConnectCodeForGame(const std::string& input)
return connectCode;
}
CEXISlippi::CEXISlippi(Core::System& system) : IEXIDevice(system)
CEXISlippi::CEXISlippi(Core::System& system, const std::string current_file_name) : IEXIDevice(system)
{
INFO_LOG_FMT(SLIPPI, "EXI SLIPPI Constructor called.");
@ -124,7 +124,7 @@ CEXISlippi::CEXISlippi(Core::System& system) : IEXIDevice(system)
g_playbackStatus = std::make_unique<SlippiPlaybackStatus>();
matchmaking = std::make_unique<SlippiMatchmaking>(user.get());
gameFileLoader = std::make_unique<SlippiGameFileLoader>();
game_reporter = std::make_unique<SlippiGameReporter>(user.get());
game_reporter = std::make_unique<SlippiGameReporter>(user.get(), current_file_name);
g_replayComm = std::make_unique<SlippiReplayComm>();
directCodes = std::make_unique<SlippiDirectCodes>("direct-codes.json");
teamsCodes = std::make_unique<SlippiDirectCodes>("teams-codes.json");
@ -2957,6 +2957,7 @@ void CEXISlippi::DMAWrite(u32 _uAddr, u32 _uSize)
g_needInputForFrame = true;
SlippiSpectateServer::getInstance().startGame();
SlippiSpectateServer::getInstance().write(&memPtr[0], receiveCommandsLen + 1);
game_reporter->PushReplayData(&memPtr[0], receiveCommandsLen + 1, "create");
}
if (byte == CMD_MENU_FRAME)
@ -2990,6 +2991,7 @@ void CEXISlippi::DMAWrite(u32 _uAddr, u32 _uSize)
writeToFileAsync(&memPtr[bufLoc], payloadLen + 1, "close");
SlippiSpectateServer::getInstance().write(&memPtr[bufLoc], payloadLen + 1);
SlippiSpectateServer::getInstance().endGame();
game_reporter->PushReplayData(&memPtr[bufLoc], payloadLen + 1, "close");
break;
case CMD_PREPARE_REPLAY:
// log.open("log.txt");
@ -3002,6 +3004,7 @@ void CEXISlippi::DMAWrite(u32 _uAddr, u32 _uSize)
g_needInputForFrame = true;
writeToFileAsync(&memPtr[bufLoc], payloadLen + 1, "");
SlippiSpectateServer::getInstance().write(&memPtr[bufLoc], payloadLen + 1);
game_reporter->PushReplayData(&memPtr[bufLoc], payloadLen + 1, "");
break;
case CMD_IS_STOCK_STEAL:
prepareIsStockSteal(&memPtr[bufLoc + 1]);
@ -3097,6 +3100,7 @@ void CEXISlippi::DMAWrite(u32 _uAddr, u32 _uSize)
default:
writeToFileAsync(&memPtr[bufLoc], payloadLen + 1, "");
SlippiSpectateServer::getInstance().write(&memPtr[bufLoc], payloadLen + 1);
game_reporter->PushReplayData(&memPtr[bufLoc], payloadLen + 1, "");
break;
}

2
Source/Core/Core/HW/EXI/EXI_DeviceSlippi.h
View file

@ -31,7 +31,7 @@ namespace ExpansionInterface
class CEXISlippi : public IEXIDevice
{
public:
CEXISlippi(Core::System& system);
CEXISlippi(Core::System& system, const std::string current_file_name);
virtual ~CEXISlippi();
void DMAWrite(u32 _uAddr, u32 _uSize) override;

4
Source/Core/Core/HW/HW.cpp
View file

@ -31,7 +31,7 @@
namespace HW
{
void Init(const Sram* override_sram)
void Init(const Sram* override_sram, const std::string current_file_name)
{
auto& system = Core::System::GetInstance();
system.GetCoreTiming().Init();
@ -44,7 +44,7 @@ void Init(const Sram* override_sram)
system.GetVideoInterface().Init();
SerialInterface::Init();
system.GetProcessorInterface().Init();
system.GetExpansionInterface().Init(override_sram); // Needs to be initialized before Memory
system.GetExpansionInterface().Init(override_sram, current_file_name); // Needs to be initialized before Memory
system.GetHSP().Init();
system.GetMemory().Init(); // Needs to be initialized before AddressSpace
AddressSpace::Init();

2
Source/Core/Core/HW/HW.h
View file

@ -8,7 +8,7 @@ struct Sram;
namespace HW
{
void Init(const Sram* override_sram);
void Init(const Sram* override_sram, const std::string current_file_name);
void Shutdown();
void DoState(PointerWrap& p);
} // namespace HW

199
Source/Core/Core/Slippi/SlippiGameReporter.cpp
View file

@ -14,31 +14,80 @@
#include <codecvt>
#include <locale>
#include <zlib.h>
#include <mbedtls/md5.h>
#include <mbedtls/md.h>
#include <json.hpp>
using json = nlohmann::json;
SlippiGameReporter::SlippiGameReporter(SlippiUser* user)
static size_t curl_receive(char *ptr, size_t size, size_t nmemb, void *rcvBuf)
{
CURL* curl = curl_easy_init();
if (curl)
size_t len = size * nmemb;
INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Received data: {}", len);
std::string *buf = (std::string *)rcvBuf;
buf->insert(buf->end(), ptr, ptr + len);
return len;
}
static size_t curl_send(char *ptr, size_t size, size_t nmemb, void *userdata)
{
std::vector<u8> *buf = (std::vector<u8> *)userdata;
INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Sending data. Size: {}, Nmemb: {}. Buffer length: {}", size, nmemb, buf->size());
size_t copy_size = size * nmemb;
if (copy_size > buf->size())
copy_size = buf->size();
if (copy_size == 0)
return 0;
// This method of reading from a vector seems so jank, im sure there's better ways to do this
memcpy(ptr, &buf->at(0), copy_size);
buf->erase(buf->begin(), buf->begin() + copy_size);
return copy_size;
}
SlippiGameReporter::SlippiGameReporter(SlippiUser *user, const std::string current_file_name)
{
CURL *curl_upload = curl_easy_init();
if (curl_upload)
{
// curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &receive);
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 10000);
curl_easy_setopt(curl_upload, CURLOPT_READFUNCTION, &curl_send);
curl_easy_setopt(curl_upload, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl_upload, CURLOPT_WRITEFUNCTION, &curl_receive);
curl_easy_setopt(curl_upload, CURLOPT_TIMEOUT_MS, 10000);
// Set up HTTP Headers
m_curl_header_list = curl_slist_append(m_curl_header_list, "Content-Type: application/json");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, m_curl_header_list);
m_curl_upload_headers = curl_slist_append(m_curl_upload_headers, "Content-Type: application/octet-stream");
curl_slist_append(m_curl_upload_headers, "Content-Encoding: gzip");
curl_slist_append(m_curl_upload_headers, "X-Goog-Content-Length-Range: 0,10000000");
curl_easy_setopt(curl_upload, CURLOPT_HTTPHEADER, m_curl_upload_headers);
#ifdef _WIN32
// ALPN support is enabled by default but requires Windows >= 8.1.
curl_easy_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, false);
curl_easy_setopt(curl_upload, CURLOPT_SSL_ENABLE_ALPN, false);
#endif
m_curl = curl;
m_curl_upload = curl_upload;
}
m_user = user;
// TODO: For mainline port, ISO file path can't be fetched this way. Look at the following:
// https://github.com/dolphin-emu/dolphin/blob/7f450f1d7e7d37bd2300f3a2134cb443d07251f9/Source/Core/Core/Movie.cpp#L246-L249;
static const mbedtls_md_info_t* s_md5_info = mbedtls_md_info_from_type(MBEDTLS_MD_MD5);
m_md5_thread = std::thread([this, current_file_name]() {
std::array<u8, 16> md5_array;
mbedtls_md_file(s_md5_info, current_file_name.c_str(), md5_array.data());
this->m_iso_hash = std::string(md5_array.begin(), md5_array.end());
INFO_LOG_FMT(SLIPPI_ONLINE, "MD5 Hash: {}", this->m_iso_hash);
});
m_md5_thread.detach();
run_thread = true;
reporting_thread = std::thread(&SlippiGameReporter::ReportThreadHandler, this);
}
@ -55,6 +104,30 @@ SlippiGameReporter::~SlippiGameReporter()
curl_slist_free_all(m_curl_header_list);
curl_easy_cleanup(m_curl);
}
if (m_curl_upload)
{
curl_slist_free_all(m_curl_upload_headers);
curl_easy_cleanup(m_curl_upload);
}
}
void SlippiGameReporter::PushReplayData(u8 *data, u32 length, std::string action) {
if (action == "create")
{
m_replay_write_idx += 1;
}
// This makes a vector at this index if it doesn't exist
auto &v = m_replay_data[m_replay_write_idx];
// Insert new data into vector
v.insert(v.end(), data, data + length);
if (action == "close")
{
m_replay_last_completed_idx = m_replay_write_idx;
}
}
void SlippiGameReporter::StartReport(GameReport report)
@ -77,6 +150,8 @@ void SlippiGameReporter::ReportThreadHandler()
// Wait for report to come in
cv.wait(lck);
auto queueHasData = !game_report_queue.empty();
// Process all messages
while (!game_report_queue.empty())
{
@ -84,12 +159,6 @@ void SlippiGameReporter::ReportThreadHandler()
game_report_queue.pop();
auto ranked = SlippiMatchmaking::OnlinePlayMode::RANKED;
auto unranked = SlippiMatchmaking::OnlinePlayMode::UNRANKED;
bool should_report = report.mode == ranked || report.mode == unranked;
if (!should_report)
{
break;
}
auto user_info = m_user->GetUserInfo();
WARN_LOG_FMT(SLIPPI_ONLINE, "Checking game report for game {}. Length: {}...", game_index,
@ -109,6 +178,7 @@ void SlippiGameReporter::ReportThreadHandler()
request["gameEndMethod"] = report.game_end_method;
request["lrasInitiator"] = report.lras_initiator;
request["stageId"] = report.stage_id;
request["isoHash"] = m_iso_hash;
json players = json::array();
for (int i = 0; i < report.players.size(); i++)
@ -131,20 +201,60 @@ void SlippiGameReporter::ReportThreadHandler()
auto requestString = request.dump();
// Send report
std::string resp;
curl_easy_setopt(m_curl, CURLOPT_POST, true);
curl_easy_setopt(m_curl, CURLOPT_URL, REPORT_URL.c_str());
curl_easy_setopt(m_curl, CURLOPT_POSTFIELDS, requestString.c_str());
curl_easy_setopt(m_curl, CURLOPT_POSTFIELDSIZE, requestString.length());
curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &resp);
CURLcode res = curl_easy_perform(m_curl);
// Increment game index even if this fails, because we don't currently retry
game_index++;
if (res != 0)
{
ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Got error executing request. Err code : {}",
static_cast<u8>(res));
Common::SleepCurrentThread(0);
continue;
}
game_index++;
long responseCode;
curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &responseCode);
if (responseCode != 200)
{
ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Server responded with non-success status: {}", responseCode);
Common::SleepCurrentThread(0);
continue;
}
// Grab resp
auto r = json::parse(resp);
bool success = r.value("success", false);
if (!success)
{
ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Report reached server but failed. {}", resp.c_str());
Common::SleepCurrentThread(0);
continue;
}
std::string uploadUrl = r.value("uploadUrl", "");
UploadReplay(m_replay_last_completed_idx, uploadUrl);
Common::SleepCurrentThread(0);
}
// Clean up replay data for games that are complete
if (queueHasData)
{
auto firstIdx = m_replay_data.begin()->first;
for (int i = firstIdx; i < m_replay_last_completed_idx; i++)
{
INFO_LOG_FMT(SLIPPI_ONLINE, "Cleaning index {} in replay data.", i);
m_replay_data[i].clear();
m_replay_data.erase(i);
}
}
}
}
@ -175,3 +285,60 @@ void SlippiGameReporter::ReportAbandonment(std::string match_id)
static_cast<u8>(res));
}
}
// https://stackoverflow.com/a/57699371/1249024
int compressToGzip(const char *input, size_t inputSize, char *output, size_t outputSize)
{
z_stream zs;
zs.zalloc = Z_NULL;
zs.zfree = Z_NULL;
zs.opaque = Z_NULL;
zs.avail_in = (uInt)inputSize;
zs.next_in = (Bytef *)input;
zs.avail_out = (uInt)outputSize;
zs.next_out = (Bytef *)output;
// hard to believe they don't have a macro for gzip encoding, "Add 16" is the best thing zlib can do:
// "Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib
// wrapper"
deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY);
deflate(&zs, Z_FINISH);
deflateEnd(&zs);
return zs.total_out;
}
void SlippiGameReporter::UploadReplay(int idx, std::string url)
{
if (url.length() <= 0)
return;
//INFO_LOG(SLIPPI_ONLINE, "Uploading replay: {}, {}", idx, url.c_str());
auto replay_data = m_replay_data[idx];
u32 raw_data_size = static_cast<u32>(replay_data.size());
u8 *rdbs = reinterpret_cast<u8 *>(&raw_data_size);
// Add header and footer to replay file
std::vector<u8> header({'{', 'U', 3, 'r', 'a', 'w', '[', '$', 'U', '#', 'l', rdbs[3], rdbs[2], rdbs[1], rdbs[0]});
replay_data.insert(replay_data.begin(), header.begin(), header.end());
std::vector<u8> footer({'U', 8, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a', '{', '}', '}'});
replay_data.insert(replay_data.end(), footer.begin(), footer.end());
std::vector<u8> gzipped_data;
gzipped_data.resize(replay_data.size());
auto res_size = compressToGzip(reinterpret_cast<char *>(&replay_data[0]), replay_data.size(),
reinterpret_cast<char *>(&gzipped_data[0]), gzipped_data.size());
gzipped_data.resize(res_size);
INFO_LOG_FMT(SLIPPI_ONLINE, "Pre-compression size: {}. Post compression size: {}", replay_data.size(), res_size);
curl_easy_setopt(m_curl_upload, CURLOPT_URL, url.c_str());
curl_easy_setopt(m_curl_upload, CURLOPT_READDATA, &gzipped_data);
curl_easy_setopt(m_curl_upload, CURLOPT_INFILESIZE, res_size);
CURLcode res = curl_easy_perform(m_curl_upload);
if (res != 0)
{
ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Got error uploading replay file. Err code: {}", static_cast<int>(res));
}
}

16
Source/Core/Core/Slippi/SlippiGameReporter.h
View file

@ -8,6 +8,7 @@
#include <string>
#include <thread>
#include <vector>
#include <map>
#include "Common/CommonTypes.h"
#include "Core/Slippi/SlippiMatchmaking.h"
#include "Core/Slippi/SlippiUser.h"
@ -37,17 +38,19 @@ public:
s8 winner_idx = 0;
u8 game_end_method = 0;
s8 lras_initiator = 0;
int stage_id;
int stage_id = 0;
std::vector<PlayerReport> players;
};
SlippiGameReporter(SlippiUser* user);
SlippiGameReporter(SlippiUser* user, const std::string current_file_name);
~SlippiGameReporter();
void StartReport(GameReport report);
void ReportAbandonment(std::string match_id);
void StartNewSession();
void ReportThreadHandler();
void PushReplayData(u8 *data, u32 length, std::string action);
void UploadReplay(int idx, std::string url);
protected:
const std::string REPORT_URL = "https://rankings-dot-slippi.uc.r.appspot.com/report";
@ -55,13 +58,22 @@ protected:
CURL* m_curl = nullptr;
struct curl_slist* m_curl_header_list = nullptr;
CURL *m_curl_upload = nullptr;
struct curl_slist *m_curl_upload_headers = nullptr;
u32 game_index = 1;
std::vector<std::string> m_player_uids;
SlippiUser* m_user;
std::string m_iso_hash;
std::queue<GameReport> game_report_queue;
std::thread reporting_thread;
std::mutex mtx;
std::condition_variable cv;
std::atomic<bool> run_thread;
std::thread m_md5_thread;
std::map<int, std::vector<u8>> m_replay_data;
int m_replay_write_idx = 0;
int m_replay_last_completed_idx = -1;
};