Refactor serialization system (#702)

* Initial work for serialization refactor

* Experiment with new naming conventions

* Mostly implement user and slot serialization.
Still needs to be fine tuned to match original implementation
Many things are left in a broken state like website features/api endpoints/lbp3 categories

* Fix release building

* Migrate scores, reviews, and more to new serialization system.
Many things are still broken but progress is steadily being made

* Fix Api responses and migrate serialization for most types

* Make serialization better and fix bugs
Fix recursive PrepareSerialization when recursive item is set during root item's PrepareSerialization, items, should be properly indexed in order but it's only tested to 1 level of recursion

* Fix review serialization

* Fix user serialization producing malformed SQL query

* Remove DefaultIfEmpty query

* MariaDB doesn't like double nested queries

* Fix LBP1 tag counter

* Implement lbp3 categories and add better deserialization handling

* Implement expression tree caching to speed up reflection and write new serializer tests

* Remove Game column from UserEntity and rename DatabaseContextModelSnapshot.cs back to DatabaseModelSnapshot.cs

* Make UserEntity username not required

* Fix recursive serialization of lists and add relevant unit tests

* Actually commit the migration

* Fix LocationTests to use new deserialization class

* Fix comments not serializing the right author username

* Replace all occurrences of StatusCode with their respective ASP.NET named result
instead of StatusCode(403) everything is now in the form of Forbid()

* Fix SlotBase.ConvertToEntity and LocationTests

* Fix compilation error

* Give Location a default value in GameUserSlot and GameUser

* Reimplement stubbed website functions

* Convert grief reports to new serialization system

* Update DatabaseModelSnapshot and bump dotnet tool version

* Remove unused directives

* Fix broken type reference

* Fix rated comments on website

* Don't include banned users in website comments

* Optimize score submission

* Fix slot id calculating in in-game comment posting

* Move serialization interfaces to types folder and add more documentation

* Allow uploading of versus scores

ProjectLighthouse.Tests/LighthouseServerTest.cs

@@ -6,7 +6,6 @@ using System.Threading.Tasks;
 using System.Xml.Serialization;
 using LBPUnion.ProjectLighthouse.Database;
 using LBPUnion.ProjectLighthouse.Helpers;
-using LBPUnion.ProjectLighthouse.Serialization;
 using LBPUnion.ProjectLighthouse.Tickets;
 using LBPUnion.ProjectLighthouse.Types.Entities.Profile;
 using LBPUnion.ProjectLighthouse.Types.Users;
@@ -38,7 +37,7 @@ public class LighthouseServerTest<TStartup> where TStartup : class
             await using DatabaseContext database = new();
             if (await database.Users.FirstOrDefaultAsync(u => u.Username == $"{username}{number}") == null)
             {
-                User user = await database.CreateUser($"{username}{number}",
+                UserEntity user = await database.CreateUser($"{username}{number}",
                     CryptoHelper.BCryptHash($"unitTestPassword{number}"));
                 user.LinkedPsnId = (ulong)number;
                 await database.SaveChangesAsync();
@@ -66,7 +65,7 @@ public class LighthouseServerTest<TStartup> where TStartup : class
     {
         HttpResponseMessage response = await this.AuthenticateResponse(number);
 
-        string responseContent = LbpSerializer.StringElement("loginResult", await response.Content.ReadAsStringAsync());
+        string responseContent = await response.Content.ReadAsStringAsync();
 
         XmlSerializer serializer = new(typeof(LoginResult));
         return (LoginResult)serializer.Deserialize(new StringReader(responseContent))!;

ProjectLighthouse.Tests/Tests/LocationTests.cs

@@ -3,6 +3,7 @@ using System.Xml.Serialization;
 using LBPUnion.ProjectLighthouse.Types.Entities.Level;
 using LBPUnion.ProjectLighthouse.Types.Entities.Profile;
 using LBPUnion.ProjectLighthouse.Types.Misc;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
 using Xunit;
 
 namespace LBPUnion.ProjectLighthouse.Tests;
@@ -17,7 +18,7 @@ public class LocationTests
             X = 1000,
             Y = 5000,
         };
-        User user = new()
+        UserEntity user = new()
         {
             Location = new Location
             {
@@ -38,7 +39,7 @@ public class LocationTests
             X = 1000,
             Y = 5000,
         };
-        Slot slot = new()
+        SlotEntity slot = new()
         {
             Location = new Location
             {
@@ -54,28 +55,30 @@ public class LocationTests
     [Fact]
     public void ShouldReadLocationAfterDeserialization()
     {
-        XmlSerializer deserializer = new(typeof(Slot));
+        XmlSerializer deserializer = new(typeof(GameUserSlot));
         const string slotData = "<slot><name>test</name><resource>test</resource><location><x>4000</x><y>9000</y></location></slot>";
 
-        Slot? deserialized = (Slot?)deserializer.Deserialize(new StringReader(slotData));
+        GameUserSlot? deserialized = (GameUserSlot?)deserializer.Deserialize(new StringReader(slotData));
         Assert.True(deserialized != null);
         Assert.True(deserialized.Name == "test");
         Assert.True(deserialized.Location.X == 4000);
         Assert.True(deserialized.Location.Y == 9000);
-        Assert.True(deserialized.LocationPacked == 17_179_869_193_000);
+        SlotEntity entity = SlotBase.ConvertToEntity(deserialized);
+        Assert.True(entity.LocationPacked == 17_179_869_193_000);
     }
 
     [Fact]
     public void ShouldDeserializeEmptyLocation()
     {
-        XmlSerializer deserializer = new(typeof(Slot));
+        XmlSerializer deserializer = new(typeof(GameUserSlot));
         const string slotData = "<slot><name>test</name></slot>";
 
-        Slot? deserialized = (Slot?)deserializer.Deserialize(new StringReader(slotData));
+        GameUserSlot? deserialized = (GameUserSlot?)deserializer.Deserialize(new StringReader(slotData));
         Assert.True(deserialized != null);
         Assert.True(deserialized.Name == "test");
         Assert.True(deserialized.Location.X == 0);
         Assert.True(deserialized.Location.Y == 0);
-        Assert.True(deserialized.LocationPacked == 0);
+        SlotEntity entity = SlotBase.ConvertToEntity(deserialized);
+        Assert.True(entity.LocationPacked == 0);
     }
 }

ProjectLighthouse.Tests/Tests/Serialization/SerializationDependencyTests.cs

@@ -0,0 +1,159 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Xml.Serialization;
+using LBPUnion.ProjectLighthouse.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Xunit;
+
+namespace LBPUnion.ProjectLighthouse.Tests.Serialization;
+
+public class SerializationDependencyTests
+{
+    private static IServiceProvider GetTestServiceProvider(params object[] dependencies)
+    {
+        ServiceCollection collection = new();
+        foreach (object o in dependencies)
+        {
+            ServiceDescriptor descriptor = new(o.GetType(), o);
+            collection.Add(descriptor);
+        }
+        return new DefaultServiceProviderFactory().CreateServiceProvider(collection);
+    }
+
+    public class TestDependency
+    {
+        public TestDependency(string secret)
+        {
+            this.Secret = secret;
+        }
+        public string Secret { get; }
+    }
+
+    public class DependencyTest : TestSerializable, INeedsPreparationForSerialization
+    {
+        [DefaultValue("")]
+        [XmlElement("secret")]
+        public string Secret { get; set; } = "";
+
+        public void PrepareSerialization(TestDependency testDependency)
+        {
+            this.Secret = testDependency.Secret;
+        }
+    }
+
+    [Fact]
+    public void ShouldInjectDependency()
+    {
+        DependencyTest serializable = new();
+        TestDependency testDependency = new("bruh");
+        string serialized = LighthouseSerializer.Serialize(GetTestServiceProvider(testDependency), serializable);
+        Assert.True(serialized == "<xmlRoot><secret>bruh</secret></xmlRoot>");
+    }
+
+    public class RecursiveDependencyTest : TestSerializable
+    {
+        [XmlElement("dependency")]
+        public DependencyTest Dependency { get; set; } = new();
+    }
+
+    [Fact]
+    public void ShouldInjectDependencyIntoNestedClass()
+    {
+        RecursiveDependencyTest serializable = new();
+        TestDependency testDependency = new("bruh");
+        string serialized = LighthouseSerializer.Serialize(GetTestServiceProvider(testDependency), serializable);
+        Assert.True(serialized == "<xmlRoot><dependency><secret>bruh</secret></dependency></xmlRoot>");
+    }
+
+    public class RecursiveDependencyTestWithPreparation : TestSerializable, INeedsPreparationForSerialization
+    {
+        [XmlElement("dependency")]
+        public DependencyTest Dependency { get; set; } = new();
+
+        [XmlElement("prepared")]
+        public string PreparedField { get; set; } = "";
+
+        public void PrepareSerialization()
+        {
+            this.PreparedField = "test";
+        }
+    }
+
+    public class DependencyOrderTest : TestSerializable, INeedsPreparationForSerialization
+    {
+        [XmlElement("dependency")]
+        public DependencyTest Dependency { get; set; } = null!;
+
+        [XmlElement("nestedDependency")]
+        public RecursiveDependencyTestWithPreparation RecursiveDependency { get; set; } = null!;
+
+        public void PrepareSerialization(TestDependency testDependency)
+        {
+            this.Dependency = new DependencyTest();
+            this.RecursiveDependency = new RecursiveDependencyTestWithPreparation();
+        }
+    }
+
+    [Fact]
+    public void ShouldDependenciesBePreparedInOrder()
+    {
+        DependencyOrderTest serializable = new();
+        TestDependency testDependency = new("bruh");
+        string serialized = LighthouseSerializer.Serialize(GetTestServiceProvider(testDependency), serializable);
+        Assert.True(serialized ==
+                    "<xmlRoot>" +
+                    "<dependency><secret>bruh</secret></dependency>" +
+                    "<nestedDependency><dependency><secret>bruh</secret></dependency><prepared>test</prepared></nestedDependency>" +
+                    "</xmlRoot>");
+    }
+
+    public class CounterDependency
+    {
+        private int counter;
+        public int Counter => this.counter++;
+
+        public CounterDependency()
+        {
+            this.counter = 0;
+        }
+
+    }
+
+    public class ListDependencyItem : ILbpSerializable, INeedsPreparationForSerialization
+    {
+        [XmlElement("counter")]
+        public int Counter { get; set; } = -1;
+
+        public void PrepareSerialization(CounterDependency counter)
+        {
+            this.Counter = counter.Counter;
+        }
+    }
+
+    public class ListDependencyTest : TestSerializable
+    {
+        [XmlElement("item")]
+        public List<ListDependencyItem> Items { get; set; } = new();
+    }
+
+    [Fact]
+    public void ShouldPrepareItemsInList()
+    {
+        ListDependencyTest serializable = new()
+        {
+            Items = new List<ListDependencyItem>
+            {
+                new(),
+                new(),
+                new(),
+            },
+        };
+        CounterDependency counterDependency = new();
+        string serialized = LighthouseSerializer.Serialize(GetTestServiceProvider(counterDependency), serializable);
+        Assert.True(serialized == "<xmlRoot><item><counter>0</counter></item><item><counter>1</counter></item><item><counter>2</counter></item></xmlRoot>");
+    }
+
+}

ProjectLighthouse.Tests/Tests/Serialization/SerializationTests.cs

@@ -0,0 +1,218 @@
+using System;
+using System.ComponentModel;
+using System.Xml.Serialization;
+using LBPUnion.ProjectLighthouse.Serialization;
+using LBPUnion.ProjectLighthouse.Types.Serialization;
+using Microsoft.Extensions.DependencyInjection;
+using Xunit;
+
+namespace LBPUnion.ProjectLighthouse.Tests.Serialization;
+
+public class TestSerializable : ILbpSerializable, IHasCustomRoot
+{
+    public virtual string GetRoot() => "xmlRoot";
+}
+
+public class SerializationTests
+{
+    private static IServiceProvider GetEmptyServiceProvider() => new DefaultServiceProviderFactory().CreateServiceProvider(new ServiceCollection());
+
+    [Fact]
+    public void ShouldNotSerializeNullObject()
+    {
+        TestSerializable? serializable = null;
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.True(serialized == "");
+    }
+
+    [Fact]
+    public void ShouldSerializeFullEmptyTag()
+    {
+        TestSerializable serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.True(serialized == "<xmlRoot></xmlRoot>");
+    }
+
+    [Fact]
+    public void ShouldSerializeWithCustomRoot()
+    {
+        TestSerializable serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.True(serialized == "<xmlRoot></xmlRoot>");
+    }
+
+    public class OverriddenRoot : TestSerializable
+    {
+        public override string GetRoot() => "xmlRoot2";
+    }
+
+    [Fact]
+    public void ShouldSerializeWithOverriddenRoot()
+    {
+        OverriddenRoot serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.True(serialized == "<xmlRoot2></xmlRoot2>");
+    }
+
+    [XmlRoot("xmlRoot3")]
+    public class RootAttribute : ILbpSerializable { }
+
+    [Fact]
+    public void ShouldSerializeWithRootAttribute()
+    {
+        RootAttribute serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.True(serialized == "<xmlRoot3></xmlRoot3>");
+    }
+
+    public class DefaultValueInt : TestSerializable
+    {
+        [DefaultValue(6)]
+        [XmlElement("defaultValueInt")]
+        public int DefaultValueTest { get; set; } = 6;
+    }
+
+    [Fact]
+    public void ShouldNotSerializeDefaultValueInt()
+    {
+        DefaultValueInt serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot></xmlRoot>");
+    }
+
+    public class NonDefaultValueInt : TestSerializable
+    {
+        [DefaultValue(6)]
+        [XmlElement("nonDefaultValueInt")]
+        public int NonDefaultValueTest { get; set; }
+    }
+
+    [Fact]
+    public void ShouldNotSerializeNonDefaultValueInt()
+    {
+        NonDefaultValueInt serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot><nonDefaultValueInt>0</nonDefaultValueInt></xmlRoot>");
+    }
+
+    public class DefaultNullableStringTest : TestSerializable
+    {
+        [DefaultValue(null)]
+        [XmlElement("defaultNullableString")]
+        public string? DefaultNullableString { get; set; }
+    }
+
+    [Fact]
+    public void ShouldNotSerializeDefaultNullableString()
+    {
+        DefaultNullableStringTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot></xmlRoot>");
+    }
+
+    public class NullNullableStringTest : TestSerializable
+    {
+        [XmlElement("nullableString")]
+        public string? NullableString { get; set; }
+    }
+
+    [Fact]
+    public void ShouldNotSerializeNullNullableString()
+    {
+        NullNullableStringTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot></xmlRoot>");
+    }
+
+    [Fact]
+    public void ShouldSerializeNonNullNullableString()
+    {
+        NullNullableStringTest serializable = new()
+        {
+            NullableString = "notNull",
+        };
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot><nullableString>notNull</nullableString></xmlRoot>");
+    }
+
+    public class NonEmptyStringTest : TestSerializable
+    {
+        [XmlElement("stringTest")]
+        public string StringTest { get; set; } = "test";
+    }
+
+    [Fact]
+    public void ShouldSerializeNonNullableString()
+    {
+        NonEmptyStringTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot><stringTest>test</stringTest></xmlRoot>");
+    }
+
+    [XmlRoot("xmlRoot", Namespace = "test")]
+    public class NameSpaceTest : TestSerializable
+    {
+        [XmlElement("test", Namespace = "test2")]
+        public int TestValue { get; set; } = 1;
+    }
+
+    [Fact]
+    public void ShouldExcludeNamespace()
+    {
+        NameSpaceTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot><test>1</test></xmlRoot>");
+    }
+
+    public class AttributeTest : TestSerializable
+    {
+        [XmlAttribute("string")]
+        public string StringAttribute { get; set; } = "test";
+        [XmlAttribute("int")]
+        public int NumberAttribute { get; set; } = 5;
+    }
+
+    [Fact]
+    public void ShouldSerializeAttributes()
+    {
+        AttributeTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized == "<xmlRoot string=\"test\" int=\"5\"></xmlRoot>");
+    }
+
+    public class NestingTest : TestSerializable
+    {
+        [XmlElement("nested")]
+        public NestedType NestedType { get; set; } = new();
+    }
+
+    public class NestedType : ILbpSerializable
+    {
+        [XmlElement("attributeTest")]
+        public AttributeTest AttributeTest { get; set; } = new();
+        [XmlElement("nestedString")]
+        public NonEmptyStringTest NonEmptyString { get; set; } = new();
+    }
+
+    [Fact]
+    public void ShouldSerializeNestedType()
+    {
+        NestingTest serializable = new();
+        string serialized = LighthouseSerializer.Serialize(GetEmptyServiceProvider(), serializable);
+        Assert.False(string.IsNullOrWhiteSpace(serialized));
+        Assert.True(serialized ==
+                    "<xmlRoot><nested>" +
+                    "<attributeTest string=\"test\" int=\"5\"></attributeTest>" +
+                    "<nestedString><stringTest>test</stringTest></nestedString>" +
+                    "</nested></xmlRoot>");
+    }
+
+}

ProjectLighthouse.Tests/Tests/SerializerTests.cs

@@ -1,42 +0,0 @@
-using System.Collections.Generic;
-using LBPUnion.ProjectLighthouse.Serialization;
-using Xunit;
-
-namespace LBPUnion.ProjectLighthouse.Tests;
-
-public class SerializerTests
-{
-    [Fact]
-    public void BlankElementWorks()
-    {
-        Assert.Equal("<test></test>", LbpSerializer.BlankElement("test"));
-    }
-
-    [Fact]
-    public void StringElementWorks()
-    {
-        Assert.Equal("<test>asd</test>", LbpSerializer.StringElement("test", "asd"));
-        Assert.Equal("<test>asd</test>", LbpSerializer.StringElement(new KeyValuePair<string, object>("test", "asd")));
-    }
-
-    [Fact]
-    public void TaggedStringElementWorks()
-    {
-        Assert.Equal("<test foo=\"bar\">asd</test>", LbpSerializer.TaggedStringElement("test", "asd", "foo", "bar"));
-        Assert.Equal
-        (
-            "<test foo=\"bar\">asd</test>",
-            LbpSerializer.TaggedStringElement(new KeyValuePair<string, object>("test", "asd"), new KeyValuePair<string, object>("foo", "bar"))
-        );
-    }
-
-    [Fact]
-    public void ElementsWorks()
-    {
-        Assert.Equal
-        (
-            "<test>asd</test><foo>bar</foo>",
-            LbpSerializer.Elements(new KeyValuePair<string, object>("test", "asd"), new KeyValuePair<string, object>("foo", "bar"))
-        );
-    }
-}