Make DB migrations use a distributed lock (#655)

* Make database migration run independent of ServerType

* Make docker-compose services not depend on gameserver

* Add debug logging and map ASP.NET folder in container

* Don't create mutex with using and manually dispose

* Adjust mysql healthcheck to make startup faster

* Make migration use a database distributed lock

* Remove debug logging

ProjectLighthouse/ProjectLighthouse.csproj

@@ -27,6 +27,7 @@
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
         <PackageReference Include="YamlDotNet" Version="12.3.1" />
         <PackageReference Include="BouncyCastle.Cryptography" Version="2.0.0" />
+        <PackageReference Include="DistributedLock.MySql" Version="1.0.1"/>
     </ItemGroup>
 
     <ItemGroup>

ProjectLighthouse/StartupTasks.cs

@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
 using System.Threading;
+using System.Threading.Tasks;
 using LBPUnion.ProjectLighthouse.Administration;
 using LBPUnion.ProjectLighthouse.Administration.Maintenance;
 using LBPUnion.ProjectLighthouse.Configuration;
@@ -15,6 +16,7 @@ using LBPUnion.ProjectLighthouse.Logging.Loggers;
 using LBPUnion.ProjectLighthouse.PlayerData.Profiles;
 using LBPUnion.ProjectLighthouse.StorableLists;
 using LBPUnion.ProjectLighthouse.Types;
+using Medallion.Threading.MySql;
 using Microsoft.EntityFrameworkCore;
 
 namespace LBPUnion.ProjectLighthouse;
@@ -59,10 +61,7 @@ public static class StartupTasks
         if (!dbConnected) Environment.Exit(1);
         using Database database = new();
         
-        #if !DEBUG
-        if (serverType == ServerType.GameServer)
-        #endif
-        migrateDatabase(database);
+        migrateDatabase(database).Wait();
 
         if (ServerConfiguration.Instance.InfluxDB.InfluxEnabled)
         {
@@ -99,7 +98,7 @@ public static class StartupTasks
         Logger.Info("Initializing repeating tasks...", LogArea.Startup);
         RepeatingTaskHandler.Initialize();
 
-            // Create admin user if no users exist
+        // Create admin user if no users exist
         if (serverType == ServerType.Website && database.Users.CountAsync().Result == 0)
         {
             const string passwordClear = "lighthouse";
@@ -154,36 +153,43 @@ public static class StartupTasks
         return didLoad;
     }
 
-    private static void migrateDatabase(Database database)
+    private static async Task migrateDatabase(Database database)
     {
+        // This mutex is used to synchronize migrations across the GameServer, Website, and Api
+        // Without it, each server would try to simultaneously migrate the database resulting in undefined behavior
+        // It is only used for startup and immediately disposed after migrating
+        Stopwatch totalStopwatch = Stopwatch.StartNew();
+        Stopwatch stopwatch = Stopwatch.StartNew();
         Logger.Info("Migrating database...", LogArea.Database);
-        Stopwatch totalStopwatch = new();
-        Stopwatch stopwatch = new();
-        totalStopwatch.Start();
-        stopwatch.Start();
-
-        database.Database.MigrateAsync().Wait();
-        stopwatch.Stop();
-        Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
-        
-        stopwatch.Reset();
-        stopwatch.Start();
-
-        List<CompletedMigration> completedMigrations = database.CompletedMigrations.ToList();
-        List<IMigrationTask> migrationsToRun = MaintenanceHelper.MigrationTasks
-            .Where(migrationTask => !completedMigrations
-                .Select(m => m.MigrationName)
-                .Contains(migrationTask.GetType().Name)
-            ).ToList();
-        
-        foreach (IMigrationTask migrationTask in migrationsToRun)
+        MySqlDistributedLock mutex = new("LighthouseMigration", ServerConfiguration.Instance.DbConnectionString);
+        await using (await mutex.AcquireAsync())
         {
-            MaintenanceHelper.RunMigration(migrationTask, database).Wait();
-        }
+            stopwatch.Stop();
+            Logger.Success($"Acquiring migration lock took {stopwatch.ElapsedMilliseconds}ms", LogArea.Database);
 
-        stopwatch.Stop();
-        totalStopwatch.Stop();
-        Logger.Success($"Extra migration tasks took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
-        Logger.Success($"Total migration took {totalStopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+            stopwatch.Restart();
+            await database.Database.MigrateAsync();
+            stopwatch.Stop();
+            Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+
+            stopwatch.Restart();
+
+            List<CompletedMigration> completedMigrations = database.CompletedMigrations.ToList();
+            List<IMigrationTask> migrationsToRun = MaintenanceHelper.MigrationTasks
+                .Where(migrationTask => !completedMigrations
+                    .Select(m => m.MigrationName)
+                    .Contains(migrationTask.GetType().Name)
+                ).ToList();
+
+            foreach (IMigrationTask migrationTask in migrationsToRun)
+            {
+                MaintenanceHelper.RunMigration(migrationTask, database).Wait();
+            }
+
+            stopwatch.Stop();
+            totalStopwatch.Stop();
+            Logger.Success($"Extra migration tasks took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+            Logger.Success($"Total migration took {totalStopwatch.ElapsedMilliseconds}ms.", LogArea.Database);
+        }
     }
 }

docker-compose.yml

@@ -23,6 +23,7 @@ services:
         condition: service_started
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   website:
     image: lighthouse:latest
     container_name: website
@@ -37,13 +38,12 @@ services:
       retries: 5
     depends_on:
       db:
-        condition: service_started
+        condition: service_healthy
       redis:
         condition: service_started
-      gameserver:
-        condition: service_healthy
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   api:
     image: lighthouse:latest
     container_name: api
@@ -58,13 +58,12 @@ services:
       retries: 5
     depends_on:
       db:
-        condition: service_started
+        condition: service_healthy
       redis:
         condition: service_started
-      gameserver:
-        condition: service_healthy
     volumes:
       - "./data:/lighthouse/data:z"
+      - "./data/.aspnet:/lighthouse/.aspnet:z"
   db:
     image: mariadb
     container_name: db
@@ -76,8 +75,9 @@ services:
       MARIADB_DATABASE: lighthouse
     healthcheck:
       test: "/usr/bin/mysql --user=root --password=lighthouse --execute \"SHOW DATABASES;\""
-      timeout: 20s
-      retries: 10
+      timeout: 10s
+      interval: 5s
+      retries: 5
     volumes:
       - "database:/var/lib/mysql"
   redis:
@@ -87,4 +87,4 @@ services:
     ports:
       - "6379:6379"
     volumes:
-      - "redis:/var/lib/redis"
+      - "redis:/var/lib/redis"