Make DB migrations use a distributed lock (#655)

* Make database migration run independent of ServerType

* Make docker-compose services not depend on gameserver

* Add debug logging and map ASP.NET folder in container

* Don't create mutex with using and manually dispose

* Adjust mysql healthcheck to make startup faster

* Make migration use a database distributed lock

* Remove debug logging

ProjectLighthouse/ProjectLighthouse.csproj

@@ -27,6 +27,7 @@
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
         <PackageReference Include="YamlDotNet" Version="12.3.1" />
         <PackageReference Include="BouncyCastle.Cryptography" Version="2.0.0" />
+        <PackageReference Include="DistributedLock.MySql" Version="1.0.1"/>
     </ItemGroup>
 
     <ItemGroup>

ProjectLighthouse/StartupTasks.cs

@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
 using System.Threading;
+using System.Threading.Tasks;
 using LBPUnion.ProjectLighthouse.Administration;
 using LBPUnion.ProjectLighthouse.Administration.Maintenance;
 using LBPUnion.ProjectLighthouse.Configuration;
@@ -15,6 +16,7 @@ using LBPUnion.ProjectLighthouse.Logging.Loggers;
 using LBPUnion.ProjectLighthouse.PlayerData.Profiles;
 using LBPUnion.ProjectLighthouse.StorableLists;
 using LBPUnion.ProjectLighthouse.Types;
+using Medallion.Threading.MySql;
 using Microsoft.EntityFrameworkCore;
 
 namespace LBPUnion.ProjectLighthouse;
@@ -59,10 +61,7 @@ public static class StartupTasks
         if (!dbConnected) Environment.Exit(1);
         using Database database = new();
         
-        #if !DEBUG
+        migrateDatabase(database).Wait();
-        if (serverType == ServerType.GameServer)
-        #endif
-        migrateDatabase(database);
 
         if (ServerConfiguration.Instance.InfluxDB.InfluxEnabled)
         {
@@ -154,20 +153,26 @@ public static class StartupTasks
         return didLoad;
     }
 
-    private static void migrateDatabase(Database database)
+    private static async Task migrateDatabase(Database database)
     {
+        // This mutex is used to synchronize migrations across the GameServer, Website, and Api
+        // Without it, each server would try to simultaneously migrate the database resulting in undefined behavior
+        // It is only used for startup and immediately disposed after migrating
+        Stopwatch totalStopwatch = Stopwatch.StartNew();
+        Stopwatch stopwatch = Stopwatch.StartNew();
         Logger.Info("Migrating database...", LogArea.Database);
-        Stopwatch totalStopwatch = new();
+        MySqlDistributedLock mutex = new("LighthouseMigration", ServerConfiguration.Instance.DbConnectionString);
-        Stopwatch stopwatch = new();
+        await using (await mutex.AcquireAsync())
-        totalStopwatch.Start();
+        {
-        stopwatch.Start();
+            stopwatch.Stop();
+            Logger.Success($"Acquiring migration lock took {stopwatch.ElapsedMilliseconds}ms", LogArea.Database);
 
-        database.Database.MigrateAsync().Wait();
+            stopwatch.Restart();
+            await database.Database.MigrateAsync();
             stopwatch.Stop();
             Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
 
-        stopwatch.Reset();
+            stopwatch.Restart();
-        stopwatch.Start();
 
             List<CompletedMigration> completedMigrations = database.CompletedMigrations.ToList();
             List<IMigrationTask> migrationsToRun = MaintenanceHelper.MigrationTasks
@@ -187,3 +192,4 @@ public static class StartupTasks
             Logger.Success($"Total migration took {totalStopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
         }
     }
+}

docker-compose.yml

@@ -23,6 +23,7 @@ services:
         condition: service_started
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   website:
     image: lighthouse:latest
     container_name: website
@@ -37,13 +38,12 @@ services:
       retries: 5
     depends_on:
       db:
-        condition: service_started
+        condition: service_healthy
       redis:
         condition: service_started
-      gameserver:
-        condition: service_healthy
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   api:
     image: lighthouse:latest
     container_name: api
@@ -58,13 +58,12 @@ services:
       retries: 5
     depends_on:
       db:
-        condition: service_started
+        condition: service_healthy
       redis:
         condition: service_started
-      gameserver:
-        condition: service_healthy
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   db:
     image: mariadb
     container_name: db
@@ -76,8 +75,9 @@ services:
       MARIADB_DATABASE: lighthouse
     healthcheck:
       test: "/usr/bin/mysql --user=root --password=lighthouse --execute \"SHOW DATABASES;\""
-      timeout: 20s
+      timeout: 10s
-      retries: 10
+      interval: 5s
+      retries: 5
     volumes:
       - "database:/var/lib/mysql"
   redis: