From f61f2ab219d08eb4cc88f5bc9137b0e4a08cd511 Mon Sep 17 00:00:00 2001 From: R2DLiu Date: Tue, 1 Aug 2023 15:45:33 -0400 Subject: [PATCH] pull in 03c57192491178b329bda001c9fb8d8ba8a28c59 --- Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp | 8 +- .../Core/Core/Slippi/SlippiGameReporter.cpp | 320 ++++++++++-------- Source/Core/Core/Slippi/SlippiGameReporter.h | 10 +- 3 files changed, 190 insertions(+), 148 deletions(-) diff --git a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp index 894c3643d7..c031f96f43 100644 --- a/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp +++ b/Source/Core/Core/HW/EXI/EXI_DeviceSlippi.cpp @@ -116,7 +116,8 @@ std::string ConvertConnectCodeForGame(const std::string& input) return connectCode; } -CEXISlippi::CEXISlippi(Core::System& system, const std::string current_file_name) : IEXIDevice(system) +CEXISlippi::CEXISlippi(Core::System& system, const std::string current_file_name) + : IEXIDevice(system) { INFO_LOG_FMT(SLIPPI, "EXI SLIPPI Constructor called."); @@ -2453,6 +2454,11 @@ void CEXISlippi::prepareOnlineMatchState() // Add the match struct block to output m_read_queue.insert(m_read_queue.end(), onlineMatchBlock.begin(), onlineMatchBlock.end()); + + // Add match id to output + std::string matchId = recentMmResult.id; + matchId.resize(51); + m_read_queue.insert(m_read_queue.end(), matchId.begin(), matchId.end()); } u16 CEXISlippi::getRandomStage() diff --git a/Source/Core/Core/Slippi/SlippiGameReporter.cpp b/Source/Core/Core/Slippi/SlippiGameReporter.cpp index 0f34cb49a0..0a1d2cab68 100644 --- a/Source/Core/Core/Slippi/SlippiGameReporter.cpp +++ b/Source/Core/Core/Slippi/SlippiGameReporter.cpp @@ -14,79 +14,81 @@ #include #include -#include -#include -#include #include +#include +#include +#include using json = nlohmann::json; -static size_t curl_receive(char *ptr, size_t size, size_t nmemb, void *rcvBuf) +static size_t curl_receive(char* ptr, size_t size, size_t nmemb, void* rcvBuf) { - size_t len = size * nmemb; - INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Received data: {}", len); + size_t len = size * nmemb; + INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Received data: {}", len); - std::string *buf = (std::string *)rcvBuf; + std::string* buf = (std::string*)rcvBuf; - buf->insert(buf->end(), ptr, ptr + len); - return len; + buf->insert(buf->end(), ptr, ptr + len); + return len; } -static size_t curl_send(char *ptr, size_t size, size_t nmemb, void *userdata) +static size_t curl_send(char* ptr, size_t size, size_t nmemb, void* userdata) { - std::vector *buf = (std::vector *)userdata; + std::vector* buf = (std::vector*)userdata; - INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Sending data. Size: {}, Nmemb: {}. Buffer length: {}", size, nmemb, buf->size()); + INFO_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Sending data. Size: {}, Nmemb: {}. Buffer length: {}", + size, nmemb, buf->size()); - size_t copy_size = size * nmemb; - if (copy_size > buf->size()) - copy_size = buf->size(); + size_t copy_size = size * nmemb; + if (copy_size > buf->size()) + copy_size = buf->size(); - if (copy_size == 0) - return 0; + if (copy_size == 0) + return 0; - // This method of reading from a vector seems so jank, im sure there's better ways to do this - memcpy(ptr, &buf->at(0), copy_size); - buf->erase(buf->begin(), buf->begin() + copy_size); + // This method of reading from a vector seems so jank, im sure there's better ways to do this + memcpy(ptr, &buf->at(0), copy_size); + buf->erase(buf->begin(), buf->begin() + copy_size); - return copy_size; + return copy_size; } -SlippiGameReporter::SlippiGameReporter(SlippiUser *user, const std::string current_file_name) +SlippiGameReporter::SlippiGameReporter(SlippiUser* user, const std::string current_file_name) { - CURL *curl_upload = curl_easy_init(); - if (curl_upload) - { - curl_easy_setopt(curl_upload, CURLOPT_READFUNCTION, &curl_send); - curl_easy_setopt(curl_upload, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(curl_upload, CURLOPT_WRITEFUNCTION, &curl_receive); - curl_easy_setopt(curl_upload, CURLOPT_TIMEOUT_MS, 10000); + CURL* curl_upload = curl_easy_init(); + if (curl_upload) + { + curl_easy_setopt(curl_upload, CURLOPT_READFUNCTION, &curl_send); + curl_easy_setopt(curl_upload, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(curl_upload, CURLOPT_WRITEFUNCTION, &curl_receive); + curl_easy_setopt(curl_upload, CURLOPT_TIMEOUT_MS, 10000); - // Set up HTTP Headers - m_curl_upload_headers = curl_slist_append(m_curl_upload_headers, "Content-Type: application/octet-stream"); - curl_slist_append(m_curl_upload_headers, "Content-Encoding: gzip"); - curl_slist_append(m_curl_upload_headers, "X-Goog-Content-Length-Range: 0,10000000"); - curl_easy_setopt(curl_upload, CURLOPT_HTTPHEADER, m_curl_upload_headers); + // Set up HTTP Headers + m_curl_upload_headers = + curl_slist_append(m_curl_upload_headers, "Content-Type: application/octet-stream"); + curl_slist_append(m_curl_upload_headers, "Content-Encoding: gzip"); + curl_slist_append(m_curl_upload_headers, "X-Goog-Content-Length-Range: 0,10000000"); + curl_easy_setopt(curl_upload, CURLOPT_HTTPHEADER, m_curl_upload_headers); #ifdef _WIN32 - // ALPN support is enabled by default but requires Windows >= 8.1. - curl_easy_setopt(curl_upload, CURLOPT_SSL_ENABLE_ALPN, false); + // ALPN support is enabled by default but requires Windows >= 8.1. + curl_easy_setopt(curl_upload, CURLOPT_SSL_ENABLE_ALPN, false); #endif - m_curl_upload = curl_upload; - } + m_curl_upload = curl_upload; + } m_user = user; // TODO: For mainline port, ISO file path can't be fetched this way. Look at the following: - // https://github.com/dolphin-emu/dolphin/blob/7f450f1d7e7d37bd2300f3a2134cb443d07251f9/Source/Core/Core/Movie.cpp#L246-L249; + // https://github.com/dolphin-emu/dolphin/blob/7f450f1d7e7d37bd2300f3a2134cb443d07251f9/Source/Core/Core/Movie.cpp#L246-L249; static const mbedtls_md_info_t* s_md5_info = mbedtls_md_info_from_type(MBEDTLS_MD_MD5); - m_md5_thread = std::thread([this, current_file_name]() { + m_md5_thread = std::thread([this, current_file_name]() { std::array md5_array; - mbedtls_md_file(s_md5_info, current_file_name.c_str(), md5_array.data()); + mbedtls_md_file(s_md5_info, current_file_name.c_str(), md5_array.data()); this->m_iso_hash = std::string(md5_array.begin(), md5_array.end()); - INFO_LOG_FMT(SLIPPI_ONLINE, "MD5 Hash: {}", this->m_iso_hash); - }); - m_md5_thread.detach(); + INFO_LOG_FMT(SLIPPI_ONLINE, "MD5 Hash: {}", this->m_iso_hash); + }); + m_md5_thread.detach(); run_thread = true; reporting_thread = std::thread(&SlippiGameReporter::ReportThreadHandler, this); @@ -106,28 +108,29 @@ SlippiGameReporter::~SlippiGameReporter() } if (m_curl_upload) - { - curl_slist_free_all(m_curl_upload_headers); - curl_easy_cleanup(m_curl_upload); - } + { + curl_slist_free_all(m_curl_upload_headers); + curl_easy_cleanup(m_curl_upload); + } } -void SlippiGameReporter::PushReplayData(u8 *data, u32 length, std::string action) { - if (action == "create") - { - m_replay_write_idx += 1; - } +void SlippiGameReporter::PushReplayData(u8* data, u32 length, std::string action) +{ + if (action == "create") + { + m_replay_write_idx += 1; + } - // This makes a vector at this index if it doesn't exist - auto &v = m_replay_data[m_replay_write_idx]; + // This makes a vector at this index if it doesn't exist + auto& v = m_replay_data[m_replay_write_idx]; - // Insert new data into vector - v.insert(v.end(), data, data + length); + // Insert new data into vector + v.insert(v.end(), data, data + length); - if (action == "close") - { - m_replay_last_completed_idx = m_replay_write_idx; - } + if (action == "close") + { + m_replay_last_completed_idx = m_replay_write_idx; + } } void SlippiGameReporter::StartReport(GameReport report) @@ -138,7 +141,7 @@ void SlippiGameReporter::StartReport(GameReport report) void SlippiGameReporter::StartNewSession() { - game_index = 1; + // Maybe we could do stuff here? We used to initialize gameIndex but that isn't required anymore } void SlippiGameReporter::ReportThreadHandler() @@ -155,14 +158,25 @@ void SlippiGameReporter::ReportThreadHandler() // Process all messages while (!game_report_queue.empty()) { - auto report = game_report_queue.front(); - game_report_queue.pop(); + auto& report = game_report_queue.front(); + report.report_attempts += 1; + + auto isFirstAttempt = report.report_attempts == 1; + auto isLastAttempt = report.report_attempts >= 5; // Only do five attempts + auto errorSleepMs = isLastAttempt ? 0 : report.report_attempts * 100; + + // If the thread is shutting down, give up after one attempt + if (!run_thread && !isFirstAttempt) + { + game_report_queue.pop(); + continue; + } auto ranked = SlippiMatchmaking::OnlinePlayMode::RANKED; auto user_info = m_user->GetUserInfo(); - WARN_LOG_FMT(SLIPPI_ONLINE, "Checking game report for game {}. Length: {}...", game_index, - report.duration_frames); + WARN_LOG_FMT(SLIPPI_ONLINE, "Checking game report for game {}. Length: {}...", + report.game_index, report.duration_frames); // Prepare report json request; @@ -170,9 +184,9 @@ void SlippiGameReporter::ReportThreadHandler() request["uid"] = user_info.uid; request["playKey"] = user_info.play_key; request["mode"] = report.mode; - request["gameIndex"] = report.mode == ranked ? report.game_index : game_index; - request["tiebreakIndex"] = report.mode == ranked ? report.tiebreak_index : 0; - request["gameIndex"] = game_index; + request["gameIndex"] = report.game_index; + request["tiebreakIndex"] = report.tiebreak_index; + request["gameIndex"] = report.game_index; request["gameDurationFrames"] = report.duration_frames; request["winnerIdx"] = report.winner_idx; request["gameEndMethod"] = report.game_end_method; @@ -197,6 +211,11 @@ void SlippiGameReporter::ReportThreadHandler() } request["players"] = players; + // Just pop before request if this is the last attempt + if (isLastAttempt) + { + game_report_queue.pop(); + } auto requestString = request.dump(); @@ -209,53 +228,67 @@ void SlippiGameReporter::ReportThreadHandler() curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &resp); CURLcode res = curl_easy_perform(m_curl); - // Increment game index even if this fails, because we don't currently retry - game_index++; - if (res != 0) { ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Got error executing request. Err code : {}", static_cast(res)); - Common::SleepCurrentThread(0); + Common::SleepCurrentThread(errorSleepMs); continue; } long responseCode; - curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &responseCode); - if (responseCode != 200) - { - ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Server responded with non-success status: {}", responseCode); - Common::SleepCurrentThread(0); - continue; - } + curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &responseCode); + if (responseCode != 200) + { + ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Server responded with non-success status: {}", + responseCode); + Common::SleepCurrentThread(errorSleepMs); + continue; + } - // Grab resp - auto r = json::parse(resp); - bool success = r.value("success", false); - if (!success) - { - ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Report reached server but failed. {}", resp.c_str()); - Common::SleepCurrentThread(0); - continue; - } + // Check if response is valid json + if (!json::accept(resp)) + { + ERROR_LOG_FMT(SLIPPI, "[GameReport] Server responded with invalid json: {}", resp); + Common::SleepCurrentThread(errorSleepMs); + continue; + } - std::string uploadUrl = r.value("uploadUrl", ""); - UploadReplay(m_replay_last_completed_idx, uploadUrl); + // Parse the response + auto r = json::parse(resp); + bool success = r.value("success", false); + if (!success) + { + ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Report reached server but failed. {}", + resp.c_str()); + Common::SleepCurrentThread(errorSleepMs); + continue; + } + + // If this was not the last attempt, pop if we are successful. On the last attempt pop will + // already have happened + if (!isLastAttempt) + { + game_report_queue.pop(); + } + + std::string uploadUrl = r.value("uploadUrl", ""); + UploadReplay(m_replay_last_completed_idx, uploadUrl); Common::SleepCurrentThread(0); } // Clean up replay data for games that are complete - if (queueHasData) - { - auto firstIdx = m_replay_data.begin()->first; - for (int i = firstIdx; i < m_replay_last_completed_idx; i++) - { - INFO_LOG_FMT(SLIPPI_ONLINE, "Cleaning index {} in replay data.", i); - m_replay_data[i].clear(); - m_replay_data.erase(i); - } - } + if (queueHasData) + { + auto firstIdx = m_replay_data.begin()->first; + for (int i = firstIdx; i < m_replay_last_completed_idx; i++) + { + INFO_LOG_FMT(SLIPPI_ONLINE, "Cleaning index {} in replay data.", i); + m_replay_data[i].clear(); + m_replay_data.erase(i); + } + } } } @@ -287,58 +320,61 @@ void SlippiGameReporter::ReportAbandonment(std::string match_id) } // https://stackoverflow.com/a/57699371/1249024 -int compressToGzip(const char *input, size_t inputSize, char *output, size_t outputSize) +int compressToGzip(const char* input, size_t inputSize, char* output, size_t outputSize) { - z_stream zs; - zs.zalloc = Z_NULL; - zs.zfree = Z_NULL; - zs.opaque = Z_NULL; - zs.avail_in = (uInt)inputSize; - zs.next_in = (Bytef *)input; - zs.avail_out = (uInt)outputSize; - zs.next_out = (Bytef *)output; + z_stream zs; + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + zs.avail_in = (uInt)inputSize; + zs.next_in = (Bytef*)input; + zs.avail_out = (uInt)outputSize; + zs.next_out = (Bytef*)output; - // hard to believe they don't have a macro for gzip encoding, "Add 16" is the best thing zlib can do: - // "Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib - // wrapper" - deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY); - deflate(&zs, Z_FINISH); - deflateEnd(&zs); - return zs.total_out; + // hard to believe they don't have a macro for gzip encoding, "Add 16" is the best thing zlib can + // do: "Add 16 to windowBits to write a simple gzip header and trailer around the compressed data + // instead of a zlib wrapper" + deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY); + deflate(&zs, Z_FINISH); + deflateEnd(&zs); + return zs.total_out; } void SlippiGameReporter::UploadReplay(int idx, std::string url) { - if (url.length() <= 0) - return; + if (url.length() <= 0) + return; - //INFO_LOG(SLIPPI_ONLINE, "Uploading replay: {}, {}", idx, url.c_str()); + // INFO_LOG(SLIPPI_ONLINE, "Uploading replay: {}, {}", idx, url.c_str()); - auto replay_data = m_replay_data[idx]; - u32 raw_data_size = static_cast(replay_data.size()); - u8 *rdbs = reinterpret_cast(&raw_data_size); + auto replay_data = m_replay_data[idx]; + u32 raw_data_size = static_cast(replay_data.size()); + u8* rdbs = reinterpret_cast(&raw_data_size); - // Add header and footer to replay file - std::vector header({'{', 'U', 3, 'r', 'a', 'w', '[', '$', 'U', '#', 'l', rdbs[3], rdbs[2], rdbs[1], rdbs[0]}); - replay_data.insert(replay_data.begin(), header.begin(), header.end()); - std::vector footer({'U', 8, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a', '{', '}', '}'}); - replay_data.insert(replay_data.end(), footer.begin(), footer.end()); + // Add header and footer to replay file + std::vector header( + {'{', 'U', 3, 'r', 'a', 'w', '[', '$', 'U', '#', 'l', rdbs[3], rdbs[2], rdbs[1], rdbs[0]}); + replay_data.insert(replay_data.begin(), header.begin(), header.end()); + std::vector footer({'U', 8, 'm', 'e', 't', 'a', 'd', 'a', 't', 'a', '{', '}', '}'}); + replay_data.insert(replay_data.end(), footer.begin(), footer.end()); - std::vector gzipped_data; - gzipped_data.resize(replay_data.size()); - auto res_size = compressToGzip(reinterpret_cast(&replay_data[0]), replay_data.size(), - reinterpret_cast(&gzipped_data[0]), gzipped_data.size()); - gzipped_data.resize(res_size); + std::vector gzipped_data; + gzipped_data.resize(replay_data.size()); + auto res_size = compressToGzip(reinterpret_cast(&replay_data[0]), replay_data.size(), + reinterpret_cast(&gzipped_data[0]), gzipped_data.size()); + gzipped_data.resize(res_size); - INFO_LOG_FMT(SLIPPI_ONLINE, "Pre-compression size: {}. Post compression size: {}", replay_data.size(), res_size); + INFO_LOG_FMT(SLIPPI_ONLINE, "Pre-compression size: {}. Post compression size: {}", + replay_data.size(), res_size); - curl_easy_setopt(m_curl_upload, CURLOPT_URL, url.c_str()); - curl_easy_setopt(m_curl_upload, CURLOPT_READDATA, &gzipped_data); - curl_easy_setopt(m_curl_upload, CURLOPT_INFILESIZE, res_size); - CURLcode res = curl_easy_perform(m_curl_upload); + curl_easy_setopt(m_curl_upload, CURLOPT_URL, url.c_str()); + curl_easy_setopt(m_curl_upload, CURLOPT_READDATA, &gzipped_data); + curl_easy_setopt(m_curl_upload, CURLOPT_INFILESIZE, res_size); + CURLcode res = curl_easy_perform(m_curl_upload); - if (res != 0) - { - ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Got error uploading replay file. Err code: {}", static_cast(res)); - } + if (res != 0) + { + ERROR_LOG_FMT(SLIPPI_ONLINE, "[GameReport] Got error uploading replay file. Err code: {}", + static_cast(res)); + } } diff --git a/Source/Core/Core/Slippi/SlippiGameReporter.h b/Source/Core/Core/Slippi/SlippiGameReporter.h index 23cb7cbacb..77be5a601b 100644 --- a/Source/Core/Core/Slippi/SlippiGameReporter.h +++ b/Source/Core/Core/Slippi/SlippiGameReporter.h @@ -3,12 +3,12 @@ #include #include // std::condition_variable #include +#include #include // std::mutex, std::unique_lock #include #include #include #include -#include #include "Common/CommonTypes.h" #include "Core/Slippi/SlippiMatchmaking.h" #include "Core/Slippi/SlippiUser.h" @@ -32,6 +32,7 @@ public: { SlippiMatchmaking::OnlinePlayMode mode = SlippiMatchmaking::OnlinePlayMode::UNRANKED; std::string match_id; + int report_attempts = 0; u32 duration_frames = 0; u32 game_index = 0; u32 tiebreak_index = 0; @@ -49,7 +50,7 @@ public: void ReportAbandonment(std::string match_id); void StartNewSession(); void ReportThreadHandler(); - void PushReplayData(u8 *data, u32 length, std::string action); + void PushReplayData(u8* data, u32 length, std::string action); void UploadReplay(int idx, std::string url); protected: @@ -58,10 +59,9 @@ protected: CURL* m_curl = nullptr; struct curl_slist* m_curl_header_list = nullptr; - CURL *m_curl_upload = nullptr; - struct curl_slist *m_curl_upload_headers = nullptr; + CURL* m_curl_upload = nullptr; + struct curl_slist* m_curl_upload_headers = nullptr; - u32 game_index = 1; std::vector m_player_uids; SlippiUser* m_user;