Migrate scores to use proper relationships (#830)

* Initial work for score migration

* Finish score migration

* Implement suggested changes from code review

* Make Score Timestamp default the current time

* Chunk insertions to reduce packet size and give all scores the same Timestamp

* Fix serialization of GameScore

* Break score ties by time then scoreId

* Make lighthouse score migration not dependent on current score implementation

ProjectLighthouse.Servers.GameServer/Controllers/Slots/ScoreController.cs

@@ -1,5 +1,4 @@
 #nullable enable
-using System.Diagnostics.CodeAnalysis;
 using LBPUnion.ProjectLighthouse.Database;
 using LBPUnion.ProjectLighthouse.Extensions;
 using LBPUnion.ProjectLighthouse.Helpers;
@@ -30,26 +29,14 @@ public class ScoreController : ControllerBase
         this.database = database;
     }
 
-    private string[] getFriendUsernames(int userId, string username)
+    private static int[] GetFriendIds(int userId)
     {
         UserFriendData? store = UserFriendStore.GetUserFriendData(userId);
-        if (store == null) return new[] { username, };
+        List<int>? friendIds = store?.FriendIds;
+        friendIds ??= new List<int>();
+        friendIds.Add(userId);
 
-        List<string> friendNames = new()
+        return friendIds.Distinct().ToArray();
-        {
-            username,
-        };
-
-        // ReSharper disable once ForeachCanBeConvertedToQueryUsingAnotherGetEnumerator
-        foreach (int friendId in store.FriendIds)
-        {
-            string? friendUsername = this.database.Users.Where(u => u.UserId == friendId)
-                .Select(u => u.Username)
-                .FirstOrDefault();
-            if (friendUsername != null) friendNames.Add(friendUsername);
-        }
-
-        return friendNames.ToArray();
     }
 
     [HttpPost("scoreboard/{slotType}/{id:int}")]
@@ -144,33 +131,33 @@ public class ScoreController : ControllerBase
 
         await this.database.SaveChangesAsync();
 
-        string playerIdCollection = string.Join(',', score.PlayerIds);
-
         ScoreEntity? existingScore = await this.database.Scores.Where(s => s.SlotId == slot.SlotId)
             .Where(s => s.ChildSlotId == 0 || s.ChildSlotId == childId)
-            .Where(s => s.PlayerIdCollection == playerIdCollection)
+            .Where(s => s.UserId == token.UserId)
             .Where(s => s.Type == score.Type)
             .FirstOrDefaultAsync();
         if (existingScore != null)
         {
             existingScore.Points = Math.Max(existingScore.Points, score.Points);
+            existingScore.Timestamp = TimeHelper.TimestampMillis;
         }
         else
         {
             ScoreEntity playerScore = new()
             {
-                PlayerIdCollection = playerIdCollection,
+                UserId = token.UserId,
                 Type = score.Type,
                 Points = score.Points,
                 SlotId = slotId,
                 ChildSlotId = childId,
+                Timestamp = TimeHelper.TimestampMillis,
             };
             this.database.Scores.Add(playerScore);
         }
 
         await this.database.SaveChangesAsync();
 
-        return this.Ok(this.getScores(new LeaderboardOptions
+        return this.Ok(await this.GetScores(new LeaderboardOptions
         {
             RootName = "scoreboardSegment",
             PageSize = 5,
@@ -178,7 +165,7 @@ public class ScoreController : ControllerBase
             SlotId = slotId,
             ChildSlotId = childId,
             ScoreType = score.Type,
-            TargetUsername = username,
+            TargetUser = token.UserId,
             TargetPlayerIds = null,
         })); 
     }
@@ -189,8 +176,6 @@ public class ScoreController : ControllerBase
     {
         GameTokenEntity token = this.GetToken();
 
-        string username = await this.database.UsernameFromGameToken(token);
-
         if (slotType == "developer") id = await SlotHelper.GetPlaceholderSlotId(this.database, id, SlotType.Developer);
 
         LeaderboardOptions options = new()
@@ -199,7 +184,7 @@ public class ScoreController : ControllerBase
             PageStart = 1,
             ScoreType = -1,
             SlotId = id,
-            TargetUsername = username,
+            TargetUser = token.UserId,
             RootName = "scoreboardSegment",
         };
         if (!HttpMethods.IsPost(this.Request.Method))
@@ -208,7 +193,7 @@ public class ScoreController : ControllerBase
             for (int i = 1; i <= 4; i++)
             {
                 options.ScoreType = i;
-                ScoreboardResponse response = this.getScores(options);
+                ScoreboardResponse response = await this.GetScores(options);
                 scoreboardResponses.Add(new PlayerScoreboardResponse(response.Scores, i));
             } 
             return this.Ok(new MultiScoreboardResponse(scoreboardResponses));
@@ -217,9 +202,9 @@ public class ScoreController : ControllerBase
         GameScore? score = await this.DeserializeBody<GameScore>();
         if (score == null) return this.BadRequest();
         options.ScoreType = score.Type;
-        options.TargetPlayerIds = this.getFriendUsernames(token.UserId, username);
+        options.TargetPlayerIds = GetFriendIds(token.UserId);
 
-        return this.Ok(this.getScores(options));
+        return this.Ok(await this.GetScores(options));
     }
 
     [HttpGet("friendscores/{slotType}/{slotId:int}/{type:int}")]
@@ -230,15 +215,13 @@ public class ScoreController : ControllerBase
 
         if (pageSize <= 0) return this.BadRequest();
 
-        string username = await this.database.UsernameFromGameToken(token);
-
         if (SlotHelper.IsTypeInvalid(slotType)) return this.BadRequest();
 
         if (slotType == "developer") slotId = await SlotHelper.GetPlaceholderSlotId(this.database, slotId, SlotType.Developer);
 
-        string[] friendIds = this.getFriendUsernames(token.UserId, username);
+        int[] friendIds = GetFriendIds(token.UserId);
 
-        return this.Ok(this.getScores(new LeaderboardOptions
+        return this.Ok(await this.GetScores(new LeaderboardOptions
         {
             RootName = "scores",
             PageSize = pageSize,
@@ -246,27 +229,24 @@ public class ScoreController : ControllerBase
             SlotId = slotId,
             ChildSlotId = childId,
             ScoreType = type,
-            TargetUsername = username,
+            TargetUser = token.UserId,
             TargetPlayerIds = friendIds,
         })); 
     }
 
     [HttpGet("topscores/{slotType}/{slotId:int}/{type:int}")]
     [HttpGet("topscores/{slotType}/{slotId:int}/{childId:int}/{type:int}")]
-    [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
     public async Task<IActionResult> TopScores(string slotType, int slotId, int? childId, int type, [FromQuery] int pageStart = -1, [FromQuery] int pageSize = 5)
     {
         GameTokenEntity token = this.GetToken();
 
         if (pageSize <= 0) return this.BadRequest();
 
-        string username = await this.database.UsernameFromGameToken(token);
-
         if (SlotHelper.IsTypeInvalid(slotType)) return this.BadRequest();
 
         if (slotType == "developer") slotId = await SlotHelper.GetPlaceholderSlotId(this.database, slotId, SlotType.Developer);
 
-        return this.Ok(this.getScores(new LeaderboardOptions
+        return this.Ok(await this.GetScores(new LeaderboardOptions
         {
             RootName = "scores",
             PageSize = pageSize,
@@ -274,52 +254,56 @@ public class ScoreController : ControllerBase
             SlotId = slotId,
             ChildSlotId = childId,
             ScoreType = type,
-            TargetUsername = username,
+            TargetUser = token.UserId,
             TargetPlayerIds = null,
         }));
     }
 
     private class LeaderboardOptions
     {
-        public int SlotId { get; set; }
+        public int SlotId { get; init; }
         public int ScoreType { get; set; }
-        public string TargetUsername { get; set; } = "";
+        public int TargetUser { get; init; }
-        public int PageStart { get; set; } = -1;
+        public int PageStart { get; init; } = -1;
-        public int PageSize { get; set; } = 5;
+        public int PageSize { get; init; } = 5;
-        public string RootName { get; set; } = "scores";
+        public string RootName { get; init; } = "scores";
-        public string[]? TargetPlayerIds;
+        public int[]? TargetPlayerIds;
         public int? ChildSlotId;
     }
 
-    private ScoreboardResponse getScores(LeaderboardOptions options)
+    private async Task<ScoreboardResponse> GetScores(LeaderboardOptions options)
     {
-        // This is hella ugly but it technically assigns the proper rank to a score
+        IQueryable<ScoreEntity> scoreQuery = this.database.Scores.Where(s => s.SlotId == options.SlotId)
-        // var needed for Anonymous type returned from SELECT
-        var rankedScores = this.database.Scores.Where(s => s.SlotId == options.SlotId)
             .Where(s => options.ScoreType == -1 || s.Type == options.ScoreType)
             .Where(s => s.ChildSlotId == 0 || s.ChildSlotId == options.ChildSlotId)
-            .AsEnumerable()
+            .Where(s => options.TargetPlayerIds == null || options.TargetPlayerIds.Contains(s.UserId));
-            .Where(s => options.TargetPlayerIds == null ||
+
-                        options.TargetPlayerIds.Any(id => s.PlayerIdCollection.Split(",").Contains(id)))
+        // First find if you have a score on a level to find scores around it
-            .OrderByDescending(s => s.Points)
+        var myScore = await scoreQuery.Where(s => s.UserId == options.TargetUser)
-            .ThenBy(s => s.ScoreId)
+            .Select(s => new
-            .ToList()
-            .Select((s, rank) => new
             {
                 Score = s,
-                Rank = rank + 1,
+                Rank = scoreQuery.Count(s2 => s2.Points > s.Points) + 1,
+            }).FirstOrDefaultAsync();
+
+        int skipAmt = options.PageStart != -1 || myScore == null ? options.PageStart - 1 : myScore.Rank - 3;
+
+        var rankedScores = scoreQuery.OrderByDescending(s => s.Points)
+            .ThenBy(s => s.Timestamp)
+            .ThenBy(s => s.ScoreId)
+            .Skip(Math.Max(0, skipAmt))
+            .Take(Math.Min(options.PageSize, 30))
+            .Select(s => new
+            {
+               Score = s,
+               Rank = scoreQuery.Count(s2 => s2.Points > s.Points) + 1,
             })
             .ToList();
 
+        int totalScores = scoreQuery.Count();
 
-        // Find your score, since even if you aren't in the top list your score is pinned
+        List<GameScore> gameScores = rankedScores.ToSerializableList(ps => GameScore.CreateFromEntity(ps.Score, ps.Rank));
-        var myScore = rankedScores.Where(rs => rs.Score.PlayerIdCollection.Split(",").Contains(options.TargetUsername)).MaxBy(rs => rs.Score.Points);
 
-        // Paginated viewing: if not requesting pageStart, get results around user
+        return new ScoreboardResponse(options.RootName, gameScores, totalScores, myScore?.Score.Points ?? 0, myScore?.Rank ?? 0);
-        var pagedScores = rankedScores.Skip(options.PageStart != -1 || myScore == null ? options.PageStart - 1 : myScore.Rank - 3).Take(Math.Min(options.PageSize, 30));
-
-        List<GameScore> gameScores = pagedScores.ToSerializableList(ps => GameScore.CreateFromEntity(ps.Score, ps.Rank));
-
-        return new ScoreboardResponse(options.RootName, gameScores, rankedScores.Count, myScore?.Score.Points ?? 0, myScore?.Rank ?? 0);
     }
 }

ProjectLighthouse.Servers.Website/Pages/Partials/LeaderboardPartial.cshtml

@@ -24,7 +24,6 @@
     @for(int i = 0; i < Model.Scores.Count; i++)
     {
         ScoreEntity score = Model.Scores[i];
-        string[] playerIds = score.PlayerIds;
         DatabaseContext database = Model.Database;          
         <div class="item">
             <span class="ui large text">
@@ -39,9 +38,9 @@
             </span>
             <div class="content">
                 <div class="list" style="padding-top: 0">
-                    @for (int j = 0; j < playerIds.Length; j++)
+                    @{
-                    {
+                        UserEntity? user = await database.Users.FindAsync(score.UserId);
-                        UserEntity? user = await database.Users.FirstOrDefaultAsync(u => u.Username == playerIds[j]);
+                    }
                         <div class="item">
                             <i class="minus icon" style="padding-top: 9px"></i>
                             <div class="content" style="padding-left: 0">
@@ -49,13 +48,8 @@
                                 {
                                     @await user.ToLink(Html, ViewData, language, timeZone)
                                 }
-                                else
-                                {
-                                    <p style="margin-top: 5px;">@playerIds[j]</p>
-                                }
                             </div>
                         </div>
-                    }
                 </div>
             </div>
         </div>

ProjectLighthouse/Administration/Maintenance/MaintenanceHelper.cs

@@ -19,15 +19,15 @@ public static class MaintenanceHelper
 {
     static MaintenanceHelper()
     {
-        Commands = getListOfInterfaceObjects<ICommand>();
+        Commands = GetListOfInterfaceObjects<ICommand>();
-        MaintenanceJobs = getListOfInterfaceObjects<IMaintenanceJob>();
+        MaintenanceJobs = GetListOfInterfaceObjects<IMaintenanceJob>();
-        MigrationTasks = getListOfInterfaceObjects<IMigrationTask>();
+        MigrationTasks = GetListOfInterfaceObjects<MigrationTask>();
-        RepeatingTasks = getListOfInterfaceObjects<IRepeatingTask>();
+        RepeatingTasks = GetListOfInterfaceObjects<IRepeatingTask>();
     }
     
     public static List<ICommand> Commands { get; }
     public static List<IMaintenanceJob> MaintenanceJobs { get; }
-    public static List<IMigrationTask> MigrationTasks { get; }
+    public static List<MigrationTask> MigrationTasks { get; }
     public static List<IRepeatingTask> RepeatingTasks { get; }
 
     public static async Task<List<LogLine>> RunCommand(IServiceProvider provider, string[] args)
@@ -80,16 +80,18 @@ public static class MaintenanceHelper
         await job.Run();
     }
 
-    public static async Task RunMigration(DatabaseContext database, IMigrationTask migrationTask)
+    public static async Task<bool> RunMigration(DatabaseContext database, MigrationTask migrationTask)
     {
-
         // Migrations should never be run twice.
-        Debug.Assert(!await database.CompletedMigrations.Has(m => m.MigrationName == migrationTask.GetType().Name));
+        Debug.Assert(!await database.CompletedMigrations.Has(m => m.MigrationName == migrationTask.GetType().Name),
+            $"Tried to run migration {migrationTask.GetType().Name} twice");
         
-        Logger.Info($"Running migration task {migrationTask.Name()}", LogArea.Database);
+        Logger.Info($"Running LH migration task {migrationTask.Name()}", LogArea.Database);
         
         bool success;
         Exception? exception = null;
+
+        Stopwatch stopwatch = Stopwatch.StartNew();
         
         try
         {
@@ -103,13 +105,14 @@ public static class MaintenanceHelper
         
         if (!success)
         {
-            Logger.Error($"Could not run migration {migrationTask.Name()}", LogArea.Database);
+            Logger.Error($"Could not run LH migration {migrationTask.Name()}", LogArea.Database);
             if (exception != null) Logger.Error(exception.ToDetailedException(), LogArea.Database);
             
-            return;
+            return false;
         }
-        
+        stopwatch.Stop();
-        Logger.Success($"Successfully completed migration {migrationTask.Name()}", LogArea.Database);
+
+        Logger.Success($"Successfully completed LH migration {migrationTask.Name()} in {stopwatch.ElapsedMilliseconds}ms", LogArea.Database);
 
         CompletedMigrationEntity completedMigration = new()
         {
@@ -119,13 +122,14 @@ public static class MaintenanceHelper
 
         database.CompletedMigrations.Add(completedMigration);
         await database.SaveChangesAsync();
+        return true;
     }
 
-    private static List<T> getListOfInterfaceObjects<T>() where T : class
+    private static List<T> GetListOfInterfaceObjects<T>() where T : class
     {
         return Assembly.GetExecutingAssembly()
             .GetTypes()
-            .Where(t => t.GetInterfaces().Contains(typeof(T)) && t.GetConstructor(Type.EmptyTypes) != null)
+            .Where(t => (t.IsSubclassOf(typeof(T)) || t.GetInterfaces().Contains(typeof(T))) && t.GetConstructor(Type.EmptyTypes) != null)
             .Select(t => Activator.CreateInstance(t) as T)
             .ToList()!;
     }

ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupBrokenVersusScoresMigration.cs

@@ -1,24 +0,0 @@
-using System.Threading.Tasks;
-using LBPUnion.ProjectLighthouse.Database;
-using LBPUnion.ProjectLighthouse.Types.Entities.Level;
-using LBPUnion.ProjectLighthouse.Types.Maintenance;
-
-namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks;
-
-public class FixBrokenVersusScores : IMigrationTask
-{
-    public string Name() => "Cleanup versus scores";
-
-    async Task<bool> IMigrationTask.Run(DatabaseContext database)
-    {
-        foreach (ScoreEntity score in database.Scores)
-        {
-            if (!score.PlayerIdCollection.Contains(':')) continue;
-
-            score.PlayerIdCollection = score.PlayerIdCollection.Replace(':', ',');
-        }
-
-        await database.SaveChangesAsync();
-        return true;
-    }
-}

ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupDuplicateScoresMigration.cs

@@ -1,51 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using LBPUnion.ProjectLighthouse.Database;
-using LBPUnion.ProjectLighthouse.Logging;
-using LBPUnion.ProjectLighthouse.Types.Entities.Level;
-using LBPUnion.ProjectLighthouse.Types.Logging;
-using LBPUnion.ProjectLighthouse.Types.Maintenance;
-
-namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks;
-
-public class CleanupDuplicateScoresMigration : IMigrationTask
-{
-    public string Name() => "Cleanup duplicate scores";
-
-    public async Task<bool> Run(DatabaseContext database)
-    {
-        List<int> duplicateScoreIds = new();
-        // The original score should have the lowest score id
-        foreach (ScoreEntity score in database.Scores.OrderBy(s => s.ScoreId)
-                     .ToList()
-                     .Where(score => !duplicateScoreIds.Contains(score.ScoreId)))
-        {
-            foreach (ScoreEntity other in database.Scores.Where(s =>
-                         s.Points == score.Points &&
-                         s.Type == score.Type &&
-                         s.SlotId == score.SlotId &&
-                         s.ScoreId != score.ScoreId &&
-                         s.ChildSlotId == score.ChildSlotId &&
-                         s.ScoreId > score.ScoreId))
-            {
-                if (score.PlayerIds.Length != other.PlayerIds.Length)
-                    continue;
-
-                HashSet<string> hashSet = new(score.PlayerIds);
-
-                if (!other.PlayerIds.All(hashSet.Contains)) continue;
-
-                Logger.Info($"Removing score with id {other.ScoreId}, slotId={other.SlotId} main='{score.PlayerIdCollection}', duplicate={other.PlayerIdCollection}", LogArea.Score);
-                database.Scores.Remove(other);
-                duplicateScoreIds.Add(other.ScoreId);
-            }
-        }
-
-        Logger.Info($"Removed a total of {duplicateScoreIds.Count} duplicate scores", LogArea.Score);
-        await database.SaveChangesAsync();
-
-        return true;
-    }
-
-}

ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSanitizedStrings.cs

@@ -8,11 +8,11 @@ using LBPUnion.ProjectLighthouse.Types.Maintenance;
 
 namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks;
 
-public class CleanupSanitizedStrings : IMigrationTask
+public class CleanupSanitizedStrings : MigrationTask
 {
-    public string Name() => "Cleanup Sanitized strings";
+    public override string Name() => "Cleanup Sanitized strings";
 
-    async Task<bool> IMigrationTask.Run(DatabaseContext database)
+    public override async Task<bool> Run(DatabaseContext database)
     {
         List<object> objsToBeSanitized = new();
 

ProjectLighthouse/Administration/Maintenance/MigrationTasks/CleanupSlotVersionMismatchMigration.cs

@@ -8,11 +8,11 @@ using LBPUnion.ProjectLighthouse.Types.Users;
 
 namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks;
 
-public class CleanupSlotVersionMismatchMigration : IMigrationTask
+public class CleanupSlotVersionMismatchMigration : MigrationTask
 {
-    public string Name() => "Cleanup slot versions";
+    public override string Name() => "Cleanup slot versions";
 
-    async Task<bool> IMigrationTask.Run(DatabaseContext database)
+    public override async Task<bool> Run(DatabaseContext database)
     {
         foreach (SlotEntity slot in database.Slots)
         {

ProjectLighthouse/Administration/Maintenance/MigrationTasks/SwitchScoreToUserIdMigration.cs

@@ -0,0 +1,234 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations.Schema;
+using System.Data;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using LBPUnion.ProjectLighthouse.Database;
+using LBPUnion.ProjectLighthouse.Helpers;
+using LBPUnion.ProjectLighthouse.Types.Maintenance;
+using Microsoft.EntityFrameworkCore;
+
+namespace LBPUnion.ProjectLighthouse.Administration.Maintenance.MigrationTasks;
+
+public class SwitchScoreToUserIdMigration : MigrationTask
+{
+    #region DB entity replication stuff
+
+    private class PostMigrationScore
+    {
+        public int ScoreId { get; set; }
+
+        public int SlotId { get; set; }
+
+        public int ChildSlotId { get; set; }
+
+        public int Type { get; set; }
+
+        public int UserId { get; set; }
+
+        public int Points { get; set; }
+
+        public long Timestamp { get; set; }
+    }
+
+    private class PreMigrationScore
+    {
+        public int ScoreId { get; set; }
+
+        public int SlotId { get; set; }
+
+        public int ChildSlotId { get; set; }
+
+        public int Type { get; set; }
+
+        public string PlayerIdCollection { get; set; }
+
+        [NotMapped]
+        public IEnumerable<string> PlayerIds => this.PlayerIdCollection.Split(",");
+
+        public int Points { get; set; }
+    }
+
+    private class MigrationUser
+    {
+        public int UserId { get; set; }
+
+        public string Username { get; set; }
+    }
+
+    #endregion
+
+    public override string Name() => "20230706020914_DropPlayerIdCollectionAndAddUserForeignKey";
+
+    public override MigrationHook HookType() => MigrationHook.Before;
+
+    private static async Task<List<PreMigrationScore>> GetAllScores(DbContext database)
+    {
+        return await MigrationHelper.GetAllObjects(database,
+            "select * from Scores",
+            reader => new PreMigrationScore
+            {
+                ScoreId = reader.GetInt32("ScoreId"),
+                SlotId = reader.GetInt32("SlotId"),
+                ChildSlotId = reader.GetInt32("ChildSlotId"),
+                Type = reader.GetInt32("Type"),
+                PlayerIdCollection = reader.GetString("PlayerIdCollection"),
+                Points = reader.GetInt32("Points"),
+            });
+    }
+
+    private static async Task<List<MigrationUser>> GetAllUsers(DbContext database)
+    {
+        return await MigrationHelper.GetAllObjects(database,
+            "select UserId, Username from Users",
+            reader => new MigrationUser
+            {
+                UserId = reader.GetInt32("UserId"),
+                Username = reader.GetString("Username"),
+            });
+    }
+
+    private static async Task<List<int>> GetAllSlots(DbContext database)
+    {
+        return await MigrationHelper.GetAllObjects(database,
+            "select SlotId from Slots",
+            reader => reader.GetInt32("SlotId"));
+    }
+
+    /// <summary>
+    /// This function deletes all existing scores and inserts the new generated scores
+    /// <para>All scores must be deleted because MySQL doesn't allow you to change primary keys</para>
+    /// </summary>
+    private static async Task ApplyFixedScores(DatabaseContext database, IReadOnlyList<PostMigrationScore> newScores)
+    {
+        // Re-order scores (The order doesn't make any difference but since we're already deleting everything we may as well) 
+        newScores = newScores.OrderByDescending(s => s.SlotId)
+            .ThenByDescending(s => s.ChildSlotId)
+            .ThenByDescending(s => s.Type)
+            .ToList();
+
+        // Set IDs for new scores
+        for (int i = 1; i < newScores.Count; i++)
+        {
+            newScores[i].ScoreId = i;
+        }
+        // Delete all existing scores
+        await database.Scores.ExecuteDeleteAsync();
+
+        long timestamp = TimeHelper.TimestampMillis;
+
+        // This is significantly faster than using standard EntityFramework Add and Save albeit a little wacky
+        foreach (PostMigrationScore[] scoreChunk in newScores.Chunk(50_000))
+        {
+            StringBuilder insertionScript = new();
+            foreach (PostMigrationScore score in scoreChunk)
+            {
+                insertionScript.AppendLine($"""
+                    insert into Scores (ScoreId, SlotId, Type, Points, ChildSlotId, Timestamp, UserId) 
+                    values('{score.ScoreId}', '{score.SlotId}', '{score.Type}', '{score.Points}', '{score.ChildSlotId}', '{timestamp}', '{score.UserId}');
+                    """);
+            }
+
+            await database.Database.ExecuteSqlRawAsync(insertionScript.ToString());
+        }
+    }
+
+    public override async Task<bool> Run(DatabaseContext database)
+    {
+        int[] scoreTypes =
+        {
+            1, 2, 3, 4, 7,
+        };
+        ConcurrentBag<PostMigrationScore> newScores = new();
+
+        List<int> slotIds = await GetAllSlots(database);
+        List<PreMigrationScore> scores = await GetAllScores(database);
+
+        // Don't run migration if there are no scores
+        if (scores == null || scores.Count == 0) return true;
+
+        List<MigrationUser> users = await GetAllUsers(database);
+
+        ConcurrentQueue<(int slotId, int type)> collection = new();
+        foreach (int slotId in slotIds.Where(id => scores.Any(score => id == score.SlotId)))
+        {
+            foreach (int type in scoreTypes)
+            {
+                collection.Enqueue((slotId, type));
+            }
+        }
+
+        ConcurrentBag<Task> taskList = new();
+        for (int i = 0; i < Environment.ProcessorCount; i++)
+        {
+            Task task = Task.Run(() =>
+            {
+                while (collection.TryDequeue(out (int slotId, int type) item))
+                {
+                    List<PostMigrationScore> fixedScores = FixScores(users,
+                            item.slotId,
+                            scores.Where(s => s.SlotId == item.slotId).Where(s => s.Type == item.type).ToList(),
+                            item.type)
+                        .ToList();
+                    fixedScores.AsParallel().ForAll(score => newScores.Add(score));
+                }
+            });
+            taskList.Add(task);
+        }
+
+        await Task.WhenAll(taskList);
+
+        await ApplyFixedScores(database, newScores.ToList());
+
+        return true;
+    }
+
+    /// <summary>
+    /// This function takes in a list of scores and creates a map of players and their highest score 
+    /// </summary>
+    private static Dictionary<string, int> CreateHighestScores(List<PreMigrationScore> scores, IReadOnlyCollection<MigrationUser> userCache)
+    {
+        Dictionary<string, int> maxPointsByPlayer = new(StringComparer.InvariantCultureIgnoreCase);
+        foreach (PreMigrationScore score in scores)
+        {
+            IEnumerable<string> players = score.PlayerIds;
+            foreach (string player in players)
+            {
+                // Remove non existent users to ensure foreign key constraint
+                if (userCache.All(u => u.Username != player)) continue;
+
+                _ = maxPointsByPlayer.TryGetValue(player, out int highestScore);
+                highestScore = Math.Max(highestScore, score.Points);
+                maxPointsByPlayer[player] = highestScore;
+            }
+        }
+
+        return maxPointsByPlayer;
+    }
+
+    /// <summary>
+    /// This function groups slots by ChildSlotId to account for adventure scores and then for each user
+    /// finds their highest score on that level and adds a new Score 
+    /// </summary>
+    private static IEnumerable<PostMigrationScore> FixScores(IReadOnlyCollection<MigrationUser> userCache, int slotId, IEnumerable<PreMigrationScore> scores, int scoreType)
+    {
+        return (
+            from slotGroup in scores.GroupBy(s => s.ChildSlotId)
+            let highestScores = CreateHighestScores(slotGroup.ToList(), userCache)
+            from kvp in highestScores
+            let userId = userCache.Where(u => u.Username == kvp.Key).Select(u => u.UserId).First()
+            select new PostMigrationScore
+            {
+                UserId = userId,
+                SlotId = slotId,
+                ChildSlotId = slotGroup.Key,
+                Points = kvp.Value,
+                // This gets set before insertion
+                Timestamp = 0L,
+                Type = scoreType,
+            }).ToList();
+    }
+}

ProjectLighthouse/Helpers/MigrationHelper.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage;
+
+namespace LBPUnion.ProjectLighthouse.Helpers;
+
+public static class MigrationHelper
+{
+    public static async Task<List<T>> GetAllObjects<T>(DbContext database, string commandText, Func<DbDataReader, T> returnFunc)
+    {
+        DbConnection dbConnection = database.Database.GetDbConnection();
+
+        await using DbCommand cmd = dbConnection.CreateCommand();
+        cmd.CommandText = commandText;
+        cmd.Transaction = GetDbTransaction(database.Database.CurrentTransaction);
+
+        await using DbDataReader reader = await cmd.ExecuteReaderAsync();
+        List<T> items = new();
+
+        if (!reader.HasRows) return default;
+
+        while (await reader.ReadAsync())
+        {
+            items.Add(returnFunc(reader));
+        }
+
+        return items;
+    }
+
+    private static DbTransaction GetDbTransaction(IDbContextTransaction dbContextTransaction)
+    {
+        if (dbContextTransaction is not IInfrastructure<DbTransaction> accessor)
+        {
+            throw new InvalidOperationException(RelationalStrings.RelationalNotInUse);
+        }
+
+        return accessor.GetInfrastructure();
+    }
+}

ProjectLighthouse/Migrations/20230628043618_AddUserIdAndTimestampToScore.cs

@@ -0,0 +1,42 @@
+using LBPUnion.ProjectLighthouse.Database;
+using LBPUnion.ProjectLighthouse.Helpers;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ProjectLighthouse.Migrations
+{
+    [DbContext(typeof(DatabaseContext))]
+    [Migration("20230628043618_AddUserIdAndTimestampToScore")]
+    public partial class AddUserIdAndTimestampToScore : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<long>(
+                name: "Timestamp",
+                table: "Scores",
+                type: "bigint",
+                nullable: false,
+                defaultValue: 0L);
+
+            migrationBuilder.AddColumn<int>(
+                name: "UserId",
+                table: "Scores",
+                type: "int",
+                nullable: false,
+                defaultValue: 0);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "Timestamp",
+                table: "Scores");
+
+            migrationBuilder.DropColumn(
+                name: "UserId",
+                table: "Scores");
+        }
+    }
+}

ProjectLighthouse/Migrations/20230706020914_DropPlayerIdCollectionAndAddUserForeignKey.cs

@@ -0,0 +1,51 @@
+using LBPUnion.ProjectLighthouse.Database;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ProjectLighthouse.Migrations
+{
+    [DbContext(typeof(DatabaseContext))]
+    [Migration("20230706020914_DropPlayerIdCollectionAndAddUserForeignKey")]
+    public partial class DropPlayerIdCollectionAndAddUserForeignKey : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "PlayerIdCollection",
+                table: "Scores");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Scores_UserId",
+                table: "Scores",
+                column: "UserId");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Scores_Users_UserId",
+                table: "Scores",
+                column: "UserId",
+                principalTable: "Users",
+                principalColumn: "UserId",
+                onDelete: ReferentialAction.Cascade);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Scores_Users_UserId",
+                table: "Scores");
+
+            migrationBuilder.DropIndex(
+                name: "IX_Scores_UserId",
+                table: "Scores");
+
+            migrationBuilder.AddColumn<string>(
+                name: "PlayerIdCollection",
+                table: "Scores",
+                type: "longtext",
+                nullable: true)
+                .Annotation("MySql:CharSet", "utf8mb4");
+        }
+    }
+}

ProjectLighthouse/ProjectLighthouse/Migrations/DatabaseModelSnapshot.cs

@@ -321,22 +321,27 @@ namespace ProjectLighthouse.Migrations
                     b.Property<int>("ChildSlotId")
                         .HasColumnType("int");
 
-                    b.Property<string>("PlayerIdCollection")
-                        .HasColumnType("longtext");
-
                     b.Property<int>("Points")
                         .HasColumnType("int");
 
                     b.Property<int>("SlotId")
                         .HasColumnType("int");
 
+                    b.Property<long>("Timestamp")
+                        .HasColumnType("bigint");
+
                     b.Property<int>("Type")
                         .HasColumnType("int");
 
+                    b.Property<int>("UserId")
+                        .HasColumnType("int");
+
                     b.HasKey("ScoreId");
 
                     b.HasIndex("SlotId");
 
+                    b.HasIndex("UserId");
+
                     b.ToTable("Scores");
                 });
 
@@ -1243,7 +1248,15 @@ namespace ProjectLighthouse.Migrations
                         .OnDelete(DeleteBehavior.Cascade)
                         .IsRequired();
 
+                    b.HasOne("LBPUnion.ProjectLighthouse.Types.Entities.Profile.UserEntity", "User")
+                        .WithMany()
+                        .HasForeignKey("UserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
                     b.Navigation("Slot");
+
+                    b.Navigation("User");
                 });
 
             modelBuilder.Entity("LBPUnion.ProjectLighthouse.Types.Entities.Level.SlotEntity", b =>

ProjectLighthouse/StartupTasks.cs

@@ -14,14 +14,17 @@ using LBPUnion.ProjectLighthouse.Helpers;
 using LBPUnion.ProjectLighthouse.Logging;
 using LBPUnion.ProjectLighthouse.Logging.Loggers;
 using LBPUnion.ProjectLighthouse.StorableLists;
-using LBPUnion.ProjectLighthouse.Types.Entities.Maintenance;
 using LBPUnion.ProjectLighthouse.Types.Entities.Profile;
 using LBPUnion.ProjectLighthouse.Types.Logging;
 using LBPUnion.ProjectLighthouse.Types.Maintenance;
-using LBPUnion.ProjectLighthouse.Types.Misc;
 using LBPUnion.ProjectLighthouse.Types.Users;
 using Medallion.Threading.MySql;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using Microsoft.Extensions.DependencyInjection;
+using ServerType = LBPUnion.ProjectLighthouse.Types.Misc.ServerType;
 
 namespace LBPUnion.ProjectLighthouse;
 
@@ -161,23 +164,53 @@ public static class StartupTasks
             Logger.Success($"Acquiring migration lock took {stopwatch.ElapsedMilliseconds}ms", LogArea.Database);
 
             stopwatch.Restart();
-            await database.Database.MigrateAsync();
+            List<string> pendingMigrations = (await database.Database.GetPendingMigrationsAsync()).ToList();
-            stopwatch.Stop();
+            IMigrator migrator = database.GetInfrastructure().GetRequiredService<IMigrator>();
-            Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+
+            async Task<bool> RunLighthouseMigrations(Func<MigrationTask, bool> predicate)
+            {
+                List<MigrationTask> tasks = MaintenanceHelper.MigrationTasks
+                    .Where(predicate)
+                    .ToList();
+                foreach (MigrationTask task in tasks)
+                {
+                    if (!await MaintenanceHelper.RunMigration(database, task)) return false;
+                }
+                return true;
+            }
+
+            Logger.Info($"There are {pendingMigrations.Count} pending migrations", LogArea.Database);
+
+            foreach (string migration in pendingMigrations)
+            {
+                try
+                {
+                    await using IDbContextTransaction transaction = await database.Database.BeginTransactionAsync();
+                    Logger.Debug($"Running migration '{migration}", LogArea.Database);
+                    stopwatch.Restart();
+                    if (!await RunLighthouseMigrations(m => m.Name() == migration && m.HookType() == MigrationHook.Before))
+                        throw new Exception($"Failed to run pre migration hook for {migration}");
+
+                    await migrator.MigrateAsync(migration);
+
+                    stopwatch.Stop();
+                    Logger.Success($"Running migration '{migration}' took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+                }
+                catch (Exception e)
+                {
+                    Logger.Error($"Failed to run migration '{migration}'", LogArea.Database);
+                    Logger.Error(e.ToDetailedException(), LogArea.Database);
+                    if (database.Database.CurrentTransaction != null)
+                        await database.Database.RollbackTransactionAsync();
+                    Environment.Exit(-1);
+                }
+            }
 
             stopwatch.Restart();
 
-            List<CompletedMigrationEntity> completedMigrations = database.CompletedMigrations.ToList();
+            List<string> completedMigrations = database.CompletedMigrations.Select(m => m.MigrationName).ToList();
-            List<IMigrationTask> migrationsToRun = MaintenanceHelper.MigrationTasks
-                .Where(migrationTask => !completedMigrations
-                    .Select(m => m.MigrationName)
-                    .Contains(migrationTask.GetType().Name)
-                ).ToList();
 
-            foreach (IMigrationTask migrationTask in migrationsToRun)
+            await RunLighthouseMigrations(m => !completedMigrations.Contains(m.GetType().Name) && m.HookType() == MigrationHook.None);
-            {
-                MaintenanceHelper.RunMigration(database, migrationTask).Wait();
-            }
 
             stopwatch.Stop();
             totalStopwatch.Stop();

ProjectLighthouse/Types/Entities/Level/ScoreEntity.cs

@@ -1,6 +1,6 @@
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
-using System.Xml.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Entities.Profile;
 
 namespace LBPUnion.ProjectLighthouse.Types.Entities.Level;
 
@@ -11,22 +11,19 @@ public class ScoreEntity
 
     public int SlotId { get; set; }
 
-    [XmlIgnore]
     [ForeignKey(nameof(SlotId))]
     public SlotEntity Slot { get; set; }
 
-    [XmlIgnore]
     public int ChildSlotId { get; set; }
 
     public int Type { get; set; }
+    
+    public int UserId { get; set; }
 
-    public string PlayerIdCollection { get; set; }
+    [ForeignKey(nameof(UserId))]
-
+    public UserEntity User { get; set; }
-    [NotMapped]
-    public string[] PlayerIds {
-        get => this.PlayerIdCollection.Split(",");
-        set => this.PlayerIdCollection = string.Join(',', value);
-    }
 
     public int Points { get; set; }
+
+    public long Timestamp { get; set; }
 }

ProjectLighthouse/Types/Entities/Maintenance/CompletedMigrationEntity.cs

@@ -5,7 +5,7 @@ using LBPUnion.ProjectLighthouse.Types.Maintenance;
 namespace LBPUnion.ProjectLighthouse.Types.Entities.Maintenance;
 
 /// <summary>
-/// A record of the completion of a <see cref="IMigrationTask"/>.
+/// A record of the completion of a <see cref="MigrationTask"/>.
 /// </summary>
 public class CompletedMigrationEntity
 {

ProjectLighthouse/Types/Maintenance/MigrationTask.cs

@@ -1,21 +1,27 @@
 using System.Threading.Tasks;
-using JetBrains.Annotations;
 using LBPUnion.ProjectLighthouse.Database;
 
 namespace LBPUnion.ProjectLighthouse.Types.Maintenance;
 
-[UsedImplicitly(ImplicitUseTargetFlags.WithInheritors)]
+public enum MigrationHook
-public interface IMigrationTask
+{
+    Before,
+    None,
+}
+
+public abstract class MigrationTask
 {
     /// <summary>
     /// The user-friendly name of a migration.
     /// </summary>
-    public string Name();
+    public abstract string Name();
-    
+
+    public virtual MigrationHook HookType() => MigrationHook.None;
+
     /// <summary>
     /// Performs the migration.
     /// </summary>
     /// <param name="database">The Lighthouse database.</param>
     /// <returns>True if successful, false if not.</returns>
-    internal Task<bool> Run(DatabaseContext database);
+    public abstract Task<bool> Run(DatabaseContext database);
 }

ProjectLighthouse/Types/Serialization/GameScore.cs

@@ -1,17 +1,24 @@
 using System.ComponentModel;
 using System.Linq;
+using System.Threading.Tasks;
 using System.Xml.Serialization;
+using LBPUnion.ProjectLighthouse.Database;
 using LBPUnion.ProjectLighthouse.Types.Entities.Level;
+using Microsoft.EntityFrameworkCore;
 
 namespace LBPUnion.ProjectLighthouse.Types.Serialization;
 
 [XmlRoot("playRecord")]
 [XmlType("playRecord")]
-public class GameScore : ILbpSerializable
+public class GameScore : ILbpSerializable, INeedsPreparationForSerialization
 {
+    [XmlIgnore]
+    public int UserId { get; set; }
+
     [XmlElement("type")]
     public int Type { get; set; }
 
+    [DefaultValue(null)]
     [XmlElement("playerIds")]
     public string[] PlayerIds;
 
@@ -26,14 +33,19 @@ public class GameScore : ILbpSerializable
     [XmlElement("score")]
     public int Points { get; set; }
 
+    public async Task PrepareSerialization(DatabaseContext database)
+    {
+        this.MainPlayer = await database.Users.Where(u => u.UserId == this.UserId)
+            .Select(u => u.Username)
+            .FirstAsync();
+    }
+
     public static GameScore CreateFromEntity(ScoreEntity entity, int rank) =>
         new()
         {
-            MainPlayer = entity.PlayerIds.ElementAtOrDefault(0) ?? "",
+            UserId = entity.UserId,
-            PlayerIds = entity.PlayerIds,
             Points = entity.Points,
             Type = entity.Type,
             Rank = rank,
         };
-
 }