Refactor serialization system (#702)

* Initial work for serialization refactor

* Experiment with new naming conventions

* Mostly implement user and slot serialization.
Still needs to be fine tuned to match original implementation
Many things are left in a broken state like website features/api endpoints/lbp3 categories

* Fix release building

* Migrate scores, reviews, and more to new serialization system.
Many things are still broken but progress is steadily being made

* Fix Api responses and migrate serialization for most types

* Make serialization better and fix bugs
Fix recursive PrepareSerialization when recursive item is set during root item's PrepareSerialization, items, should be properly indexed in order but it's only tested to 1 level of recursion

* Fix review serialization

* Fix user serialization producing malformed SQL query

* Remove DefaultIfEmpty query

* MariaDB doesn't like double nested queries

* Fix LBP1 tag counter

* Implement lbp3 categories and add better deserialization handling

* Implement expression tree caching to speed up reflection and write new serializer tests

* Remove Game column from UserEntity and rename DatabaseContextModelSnapshot.cs back to DatabaseModelSnapshot.cs

* Make UserEntity username not required

* Fix recursive serialization of lists and add relevant unit tests

* Actually commit the migration

* Fix LocationTests to use new deserialization class

* Fix comments not serializing the right author username

* Replace all occurrences of StatusCode with their respective ASP.NET named result
instead of StatusCode(403) everything is now in the form of Forbid()

* Fix SlotBase.ConvertToEntity and LocationTests

* Fix compilation error

* Give Location a default value in GameUserSlot and GameUser

* Reimplement stubbed website functions

* Convert grief reports to new serialization system

* Update DatabaseModelSnapshot and bump dotnet tool version

* Remove unused directives

* Fix broken type reference

* Fix rated comments on website

* Don't include banned users in website comments

* Optimize score submission

* Fix slot id calculating in in-game comment posting

* Move serialization interfaces to types folder and add more documentation

* Allow uploading of versus scores

ProjectLighthouse.Servers.GameServer/Types/Categories/CategoryHelper.cs

@@ -20,6 +20,6 @@ public static class CategoryHelper
         Categories.Add(new LuckyDipCategory());
 
         using DatabaseContext database = new();
-        foreach (DatabaseCategory category in database.CustomCategories) Categories.Add(new CustomCategory(category));
+        foreach (DatabaseCategoryEntity category in database.CustomCategories) Categories.Add(new CustomCategory(category));
     }
 }

ProjectLighthouse.Servers.GameServer/Types/Categories/CategoryWithUser.cs

@@ -2,18 +2,19 @@
 using System.Diagnostics;
 using LBPUnion.ProjectLighthouse.Database;
 using LBPUnion.ProjectLighthouse.Logging;
-using LBPUnion.ProjectLighthouse.Serialization;
 using LBPUnion.ProjectLighthouse.Types.Entities.Level;
 using LBPUnion.ProjectLighthouse.Types.Entities.Profile;
 using LBPUnion.ProjectLighthouse.Types.Levels;
 using LBPUnion.ProjectLighthouse.Types.Logging;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Users;
 
 namespace LBPUnion.ProjectLighthouse.Servers.GameServer.Types.Categories;
 
 public abstract class CategoryWithUser : Category
 {
-    public abstract Slot? GetPreviewSlot(DatabaseContext database, User user);
-    public override Slot? GetPreviewSlot(DatabaseContext database)
+    public abstract SlotEntity? GetPreviewSlot(DatabaseContext database, UserEntity user);
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database)
     {
         #if DEBUG
         Logger.Error("tried to get preview slot without user on CategoryWithUser", LogArea.Category);
@@ -22,7 +23,7 @@ public abstract class CategoryWithUser : Category
         return null;
     }
 
-    public abstract int GetTotalSlots(DatabaseContext database, User user);
+    public abstract int GetTotalSlots(DatabaseContext database, UserEntity user);
     public override int GetTotalSlots(DatabaseContext database)
     {
         #if DEBUG
@@ -32,14 +33,14 @@ public abstract class CategoryWithUser : Category
         return -1;
     }
 
-    public abstract IQueryable<Slot> GetSlots(DatabaseContext database, User user, int pageStart, int pageSize);
-    public override IList<Slot> GetSlots(DatabaseContext database, int pageStart, int pageSize)
+    public abstract IQueryable<SlotEntity> GetSlots(DatabaseContext database, UserEntity user, int pageStart, int pageSize);
+    public override IList<SlotEntity> GetSlots(DatabaseContext database, int pageStart, int pageSize)
     {
         #if DEBUG
         Logger.Error("tried to get slots without user on CategoryWithUser", LogArea.Category);
         if (Debugger.IsAttached) Debugger.Break();
         #endif
-        return new List<Slot>();
+        return new List<SlotEntity>();
     }
 
     public new string Serialize(DatabaseContext database)
@@ -48,35 +49,13 @@ public abstract class CategoryWithUser : Category
         return string.Empty;
     }
 
-    public string Serialize(DatabaseContext database, User user)
+    public GameCategory Serialize(DatabaseContext database, UserEntity user)
     {
-        Slot? previewSlot = this.GetPreviewSlot(database, user);
-
-        string previewResults = "";
-        if (previewSlot != null)
-            previewResults = LbpSerializer.TaggedStringElement
-            (
-                "results",
-                previewSlot.Serialize(),
-                new Dictionary<string, object>
-                {
-                    {
-                        "total", this.GetTotalSlots(database, user)
-                    },
-                    {
-                        "hint_start", "2"
-                    },
-                }
-            );
-
-        return LbpSerializer.StringElement
-        (
-            "category",
-            LbpSerializer.StringElement("name", this.Name) +
-            LbpSerializer.StringElement("description", this.Description) +
-            LbpSerializer.StringElement("url", this.IngameEndpoint) +
-            (previewSlot == null ? "" : previewResults) +
-            LbpSerializer.StringElement("icon", this.IconHash)
-        );
+        List<SlotBase> slots = new()
+        {
+            SlotBase.CreateFromEntity(this.GetPreviewSlot(database, user), GameVersion.LittleBigPlanet3, user.UserId),
+        };
+        int totalSlots = this.GetTotalSlots(database, user);
+        return GameCategory.CreateFromEntity(this, new GenericSlotResponse(slots, totalSlots, 2));
     }
 }

ProjectLighthouse.Servers.GameServer/Types/Categories/CustomCategory.cs

@@ -21,7 +21,7 @@ public class CustomCategory : Category
         this.SlotIds = slotIds.ToList();
     }
 
-    public CustomCategory(DatabaseCategory category)
+    public CustomCategory(DatabaseCategoryEntity category)
     {
         this.Name = category.Name;
         this.Description = category.Description;
@@ -35,8 +35,8 @@ public class CustomCategory : Category
     public sealed override string Description { get; set; }
     public sealed override string IconHash { get; set; }
     public sealed override string Endpoint { get; set; }
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.FirstOrDefault(s => s.SlotId == this.SlotIds[0]);
-    public override IQueryable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.FirstOrDefault(s => s.SlotId == this.SlotIds[0]);
+    public override IQueryable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3).Where(s => this.SlotIds.Contains(s.SlotId));
     public override int GetTotalSlots(DatabaseContext database) => this.SlotIds.Count;

ProjectLighthouse.Servers.GameServer/Types/Categories/HeartedCategory.cs

@@ -15,7 +15,7 @@ public class HeartedCategory : CategoryWithUser
     public override string Description { get; set; } = "Content you've hearted";
     public override string IconHash { get; set; } = "g820611";
     public override string Endpoint { get; set; } = "hearted";
-    public override Slot? GetPreviewSlot(DatabaseContext database, User user) // note: developer slots act up in LBP3 when listed here, so I omitted it
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database, UserEntity user) // note: developer slots act up in LBP3 when listed here, so I omitted it
         => database.HeartedLevels.Where(h => h.UserId == user.UserId)
             .Where(h => h.Slot.Type == SlotType.User && !h.Slot.Hidden && h.Slot.GameVersion <= GameVersion.LittleBigPlanet3)
             .OrderByDescending(h => h.HeartedLevelId)
@@ -24,7 +24,7 @@ public class HeartedCategory : CategoryWithUser
             .ByGameVersion(GameVersion.LittleBigPlanet3, false, false, true)
             .FirstOrDefault();
 
-    public override IQueryable<Slot> GetSlots(DatabaseContext database, User user, int pageStart, int pageSize)
+    public override IQueryable<SlotEntity> GetSlots(DatabaseContext database, UserEntity user, int pageStart, int pageSize)
         => database.HeartedLevels.Where(h => h.UserId == user.UserId)
             .Where(h => h.Slot.Type == SlotType.User && !h.Slot.Hidden && h.Slot.GameVersion <= GameVersion.LittleBigPlanet3)
             .OrderByDescending(h => h.HeartedLevelId)
@@ -34,5 +34,5 @@ public class HeartedCategory : CategoryWithUser
             .Skip(Math.Max(0, pageStart))
             .Take(Math.Min(pageSize, 20));
 
-    public override int GetTotalSlots(DatabaseContext database, User user) => database.HeartedLevels.Count(h => h.UserId == user.UserId);
+    public override int GetTotalSlots(DatabaseContext database, UserEntity user) => database.HeartedLevels.Count(h => h.UserId == user.UserId);
 }

ProjectLighthouse.Servers.GameServer/Types/Categories/HighestRatedCategory.cs

@@ -14,8 +14,8 @@ public class HighestRatedCategory : Category
     public override string Description { get; set; } = "Community Highest Rated content";
     public override string IconHash { get; set; } = "g820603";
     public override string Endpoint { get; set; } = "thumbs";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).AsEnumerable().MaxBy(s => s.Thumbsup);
-    public override IEnumerable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).AsEnumerable().MaxBy(s => s.Thumbsup);
+    public override IEnumerable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .AsEnumerable()

ProjectLighthouse.Servers.GameServer/Types/Categories/LuckyDipCategory.cs

@@ -14,8 +14,8 @@ public class LuckyDipCategory : Category
     public override string Description { get; set; } = "Randomized uploaded content";
     public override string IconHash { get; set; } = "g820605";
     public override string Endpoint { get; set; } = "lbp2luckydip";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).OrderByDescending(_ => EF.Functions.Random()).FirstOrDefault();
-    public override IQueryable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).OrderByDescending(_ => EF.Functions.Random()).FirstOrDefault();
+    public override IQueryable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .OrderByDescending(_ => EF.Functions.Random())

ProjectLighthouse.Servers.GameServer/Types/Categories/MostHeartedCategory.cs

@@ -14,8 +14,8 @@ public class MostHeartedCategory : Category
     public override string Description { get; set; } = "The Most Hearted Content";
     public override string IconHash { get; set; } = "g820607";
     public override string Endpoint { get; set; } = "mostHearted";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).AsEnumerable().MaxBy(s => s.Hearts);
-    public override IEnumerable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).AsEnumerable().MaxBy(s => s.Hearts);
+    public override IEnumerable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .AsEnumerable()

ProjectLighthouse.Servers.GameServer/Types/Categories/MostPlayedCategory.cs

@@ -13,12 +13,12 @@ public class MostPlayedCategory : Category
     public override string Description { get; set; } = "The most played content";
     public override string IconHash { get; set; } = "g820608";
     public override string Endpoint { get; set; } = "mostUniquePlays";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots
         .Where(s => s.Type == SlotType.User)
         .OrderByDescending(s => s.PlaysLBP1Unique + s.PlaysLBP2Unique + s.PlaysLBP3Unique)
         .ThenByDescending(s => s.PlaysLBP1 + s.PlaysLBP2 + s.PlaysLBP3)
         .FirstOrDefault();
-    public override IQueryable<Slot> GetSlots
+    public override IQueryable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .OrderByDescending(s => s.PlaysLBP1Unique + s.PlaysLBP2Unique + s.PlaysLBP3Unique)

ProjectLighthouse.Servers.GameServer/Types/Categories/NewestLevelsCategory.cs

@@ -13,8 +13,8 @@ public class NewestLevelsCategory : Category
     public override string Description { get; set; } = "The most recently published content";
     public override string IconHash { get; set; } = "g820623";
     public override string Endpoint { get; set; } = "newest";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).OrderByDescending(s => s.FirstUploaded).FirstOrDefault();
-    public override IQueryable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.Where(s => s.Type == SlotType.User).OrderByDescending(s => s.FirstUploaded).FirstOrDefault();
+    public override IQueryable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .OrderByDescending(s => s.FirstUploaded)

ProjectLighthouse.Servers.GameServer/Types/Categories/QueueCategory.cs

@@ -15,7 +15,7 @@ public class QueueCategory : CategoryWithUser
     public override string Description { get; set; } = "Your queued content";
     public override string IconHash { get; set; } = "g820614";
     public override string Endpoint { get; set; } = "queue";
-    public override Slot? GetPreviewSlot(DatabaseContext database, User user)
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database, UserEntity user)
         => database.QueuedLevels.Where(q => q.UserId == user.UserId)
             .Where(q => q.Slot.Type == SlotType.User && !q.Slot.Hidden && q.Slot.GameVersion <= GameVersion.LittleBigPlanet3)
             .OrderByDescending(q => q.QueuedLevelId)
@@ -24,7 +24,7 @@ public class QueueCategory : CategoryWithUser
             .ByGameVersion(GameVersion.LittleBigPlanet3, false, false, true)
             .FirstOrDefault();
 
-    public override IQueryable<Slot> GetSlots(DatabaseContext database, User user, int pageStart, int pageSize)
+    public override IQueryable<SlotEntity> GetSlots(DatabaseContext database, UserEntity user, int pageStart, int pageSize)
         => database.QueuedLevels.Where(q => q.UserId == user.UserId)
             .Where(q => q.Slot.Type == SlotType.User && !q.Slot.Hidden && q.Slot.GameVersion <= GameVersion.LittleBigPlanet3)
             .OrderByDescending(q => q.QueuedLevelId)
@@ -34,5 +34,5 @@ public class QueueCategory : CategoryWithUser
             .Skip(Math.Max(0, pageStart - 1))
             .Take(Math.Min(pageSize, 20));
 
-    public override int GetTotalSlots(DatabaseContext database, User user) => database.QueuedLevels.Count(q => q.UserId == user.UserId);
+    public override int GetTotalSlots(DatabaseContext database, UserEntity user) => database.QueuedLevels.Count(q => q.UserId == user.UserId);
 }

ProjectLighthouse.Servers.GameServer/Types/Categories/TeamPicksCategory.cs

@@ -13,8 +13,8 @@ public class TeamPicksCategory : Category
     public override string Description { get; set; } = "Community Team Picks";
     public override string IconHash { get; set; } = "g820626";
     public override string Endpoint { get; set; } = "team_picks";
-    public override Slot? GetPreviewSlot(DatabaseContext database) => database.Slots.OrderByDescending(s => s.FirstUploaded).FirstOrDefault(s => s.TeamPick);
-    public override IQueryable<Slot> GetSlots
+    public override SlotEntity? GetPreviewSlot(DatabaseContext database) => database.Slots.OrderByDescending(s => s.FirstUploaded).FirstOrDefault(s => s.TeamPick);
+    public override IQueryable<SlotEntity> GetSlots
         (DatabaseContext database, int pageStart, int pageSize)
         => database.Slots.ByGameVersion(GameVersion.LittleBigPlanet3, false, true)
             .OrderByDescending(s => s.FirstUploaded)

ProjectLighthouse.Servers.GameServer/Types/Misc/ResourceList.cs

@@ -1,4 +1,5 @@
 using System.Xml.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
 
 namespace LBPUnion.ProjectLighthouse.Servers.GameServer.Types.Misc;
 
@@ -9,7 +10,7 @@ namespace LBPUnion.ProjectLighthouse.Servers.GameServer.Types.Misc;
 /// </summary>
 [XmlRoot("resources")]
 [XmlType("resources")]
-public class ResourceList
+public class ResourceList : ILbpSerializable
 {
     [XmlElement("resource")]
     public string[]? Resources;

ProjectLighthouse.Servers.GameServer/Types/Users/ClientsConnected.cs

@@ -1,25 +0,0 @@
-using LBPUnion.ProjectLighthouse.Serialization;
-using Microsoft.EntityFrameworkCore;
-
-namespace LBPUnion.ProjectLighthouse.Servers.GameServer.Types.Users;
-
-[Keyless]
-public class ClientsConnected
-{
-    public bool Lbp1 { get; set; }
-    public bool Lbp2 { get; set; }
-    public bool LbpMe { get; set; }
-    public bool Lbp3Ps3 { get; set; }
-    public bool Lbp3Ps4 { get; set; }
-
-    public string Serialize()
-        => LbpSerializer.StringElement
-        (
-            "clientsConnected",
-            LbpSerializer.StringElement("lbp1", this.Lbp1) +
-            LbpSerializer.StringElement("lbp2", this.Lbp2) +
-            LbpSerializer.StringElement("lbpme", this.LbpMe) +
-            LbpSerializer.StringElement("lbp3ps3", this.Lbp3Ps3) +
-            LbpSerializer.StringElement("lbp3ps4", this.Lbp3Ps4)
-        );
-}

ProjectLighthouse.Servers.GameServer/Types/Users/PrivacySettings.cs

@@ -1,24 +1,16 @@
 #nullable enable
 using System.Xml.Serialization;
-using LBPUnion.ProjectLighthouse.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
 
 namespace LBPUnion.ProjectLighthouse.Servers.GameServer.Types.Users;
 
 [XmlRoot("privacySettings")]
 [XmlType("privacySettings")]
-public class PrivacySettings
+public class PrivacySettings : ILbpSerializable
 {
     [XmlElement("levelVisiblity")]
     public string? LevelVisibility { get; set; }
 
     [XmlElement("profileVisiblity")]
     public string? ProfileVisibility { get; set; }
-
-    public string Serialize()
-        => LbpSerializer.StringElement
-        (
-            "privacySettings",
-            LbpSerializer.StringElement("levelVisibility", this.LevelVisibility) +
-            LbpSerializer.StringElement("profileVisibility", this.ProfileVisibility)
-        );
 }