From d59fd000c34251e7702eac676039bac2b7f88e3d Mon Sep 17 00:00:00 2001 From: Josh Date: Sat, 4 Feb 2023 23:29:22 -0600 Subject: [PATCH] Make DB migrations use a distributed lock (#655) * Make database migration run independent of ServerType * Make docker-compose services not depend on gameserver * Add debug logging and map ASP.NET folder in container * Don't create mutex with using and manually dispose * Adjust mysql healthcheck to make startup faster * Make migration use a database distributed lock * Remove debug logging --- ProjectLighthouse/ProjectLighthouse.csproj | 1 + ProjectLighthouse/StartupTasks.cs | 70 ++++++++++++---------- docker-compose.yml | 18 +++--- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/ProjectLighthouse/ProjectLighthouse.csproj b/ProjectLighthouse/ProjectLighthouse.csproj index ba5911a8..dc8d9ccd 100644 --- a/ProjectLighthouse/ProjectLighthouse.csproj +++ b/ProjectLighthouse/ProjectLighthouse.csproj @@ -27,6 +27,7 @@ + diff --git a/ProjectLighthouse/StartupTasks.cs b/ProjectLighthouse/StartupTasks.cs index 45151680..65152873 100644 --- a/ProjectLighthouse/StartupTasks.cs +++ b/ProjectLighthouse/StartupTasks.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Linq; using System.Reflection; using System.Threading; +using System.Threading.Tasks; using LBPUnion.ProjectLighthouse.Administration; using LBPUnion.ProjectLighthouse.Administration.Maintenance; using LBPUnion.ProjectLighthouse.Configuration; @@ -15,6 +16,7 @@ using LBPUnion.ProjectLighthouse.Logging.Loggers; using LBPUnion.ProjectLighthouse.PlayerData.Profiles; using LBPUnion.ProjectLighthouse.StorableLists; using LBPUnion.ProjectLighthouse.Types; +using Medallion.Threading.MySql; using Microsoft.EntityFrameworkCore; namespace LBPUnion.ProjectLighthouse; @@ -59,10 +61,7 @@ public static class StartupTasks if (!dbConnected) Environment.Exit(1); using Database database = new(); - #if !DEBUG - if (serverType == ServerType.GameServer) - #endif - migrateDatabase(database); + migrateDatabase(database).Wait(); if (ServerConfiguration.Instance.InfluxDB.InfluxEnabled) { @@ -99,7 +98,7 @@ public static class StartupTasks Logger.Info("Initializing repeating tasks...", LogArea.Startup); RepeatingTaskHandler.Initialize(); - // Create admin user if no users exist + // Create admin user if no users exist if (serverType == ServerType.Website && database.Users.CountAsync().Result == 0) { const string passwordClear = "lighthouse"; @@ -154,36 +153,43 @@ public static class StartupTasks return didLoad; } - private static void migrateDatabase(Database database) + private static async Task migrateDatabase(Database database) { + // This mutex is used to synchronize migrations across the GameServer, Website, and Api + // Without it, each server would try to simultaneously migrate the database resulting in undefined behavior + // It is only used for startup and immediately disposed after migrating + Stopwatch totalStopwatch = Stopwatch.StartNew(); + Stopwatch stopwatch = Stopwatch.StartNew(); Logger.Info("Migrating database...", LogArea.Database); - Stopwatch totalStopwatch = new(); - Stopwatch stopwatch = new(); - totalStopwatch.Start(); - stopwatch.Start(); - - database.Database.MigrateAsync().Wait(); - stopwatch.Stop(); - Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); - - stopwatch.Reset(); - stopwatch.Start(); - - List completedMigrations = database.CompletedMigrations.ToList(); - List migrationsToRun = MaintenanceHelper.MigrationTasks - .Where(migrationTask => !completedMigrations - .Select(m => m.MigrationName) - .Contains(migrationTask.GetType().Name) - ).ToList(); - - foreach (IMigrationTask migrationTask in migrationsToRun) + MySqlDistributedLock mutex = new("LighthouseMigration", ServerConfiguration.Instance.DbConnectionString); + await using (await mutex.AcquireAsync()) { - MaintenanceHelper.RunMigration(migrationTask, database).Wait(); - } + stopwatch.Stop(); + Logger.Success($"Acquiring migration lock took {stopwatch.ElapsedMilliseconds}ms", LogArea.Database); - stopwatch.Stop(); - totalStopwatch.Stop(); - Logger.Success($"Extra migration tasks took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); - Logger.Success($"Total migration took {totalStopwatch.ElapsedMilliseconds}ms.", LogArea.Database); + stopwatch.Restart(); + await database.Database.MigrateAsync(); + stopwatch.Stop(); + Logger.Success($"Structure migration took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); + + stopwatch.Restart(); + + List completedMigrations = database.CompletedMigrations.ToList(); + List migrationsToRun = MaintenanceHelper.MigrationTasks + .Where(migrationTask => !completedMigrations + .Select(m => m.MigrationName) + .Contains(migrationTask.GetType().Name) + ).ToList(); + + foreach (IMigrationTask migrationTask in migrationsToRun) + { + MaintenanceHelper.RunMigration(migrationTask, database).Wait(); + } + + stopwatch.Stop(); + totalStopwatch.Stop(); + Logger.Success($"Extra migration tasks took {stopwatch.ElapsedMilliseconds}ms.", LogArea.Database); + Logger.Success($"Total migration took {totalStopwatch.ElapsedMilliseconds}ms.", LogArea.Database); + } } } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d84ba34d..2894ce8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: condition: service_started volumes: - "./data:/lighthouse/data:z" + - "./data/.aspnet:/lighthouse/.aspnet:z" website: image: lighthouse:latest container_name: website @@ -37,13 +38,12 @@ services: retries: 5 depends_on: db: - condition: service_started + condition: service_healthy redis: condition: service_started - gameserver: - condition: service_healthy volumes: - "./data:/lighthouse/data:z" + - "./data/.aspnet:/lighthouse/.aspnet:z" api: image: lighthouse:latest container_name: api @@ -58,13 +58,12 @@ services: retries: 5 depends_on: db: - condition: service_started + condition: service_healthy redis: condition: service_started - gameserver: - condition: service_healthy volumes: - "./data:/lighthouse/data:z" + - "./data/.aspnet:/lighthouse/.aspnet:z" db: image: mariadb container_name: db @@ -76,8 +75,9 @@ services: MARIADB_DATABASE: lighthouse healthcheck: test: "/usr/bin/mysql --user=root --password=lighthouse --execute \"SHOW DATABASES;\"" - timeout: 20s - retries: 10 + timeout: 10s + interval: 5s + retries: 5 volumes: - "database:/var/lib/mysql" redis: @@ -87,4 +87,4 @@ services: ports: - "6379:6379" volumes: - - "redis:/var/lib/redis" + - "redis:/var/lib/redis" \ No newline at end of file