Retry logic for migrations

services/administration/src/EShopOnAbp.AdministrationService.HttpApi.Host/AdministrationServiceHttpApiHostModule.cs

@@ -103,6 +103,6 @@ public class AdministrationServiceHttpApiHostModule : AbpModule
     {
         await context.ServiceProvider
             .GetRequiredService<AdministrationServiceDatabaseMigrationChecker>()
-            .CheckAndApplyDatabaseMigrations();
+            .CheckAndApplyDatabaseMigrationsAsync();
     }
 }

services/administration/src/EShopOnAbp.AdministrationService.HttpApi.Host/DbMigrations/AdministrationServiceDatabaseMigrationChecker.cs

@@ -38,11 +38,11 @@ public class AdministrationServiceDatabaseMigrationChecker
         _permissionDataSeeder = permissionDataSeeder;
     }
 
-    public override async Task CheckAndApplyDatabaseMigrations()
+    public override async Task CheckAndApplyDatabaseMigrationsAsync()
     {
-        await base.CheckAndApplyDatabaseMigrations();
+        await base.CheckAndApplyDatabaseMigrationsAsync();
 
-        await SeedDataAsync();
+        await TryAsync(async () => await SeedDataAsync());
     }
 
     private async Task SeedDataAsync()

services/identity/src/EShopOnAbp.IdentityService.HttpApi.Host/DbMigrations/IdentityServiceDatabaseMigrationChecker.cs

@@ -12,14 +12,11 @@ namespace EShopOnAbp.IdentityService.DbMigrations;
 
 public class IdentityServiceDatabaseMigrationChecker : PendingEfCoreMigrationsChecker<IdentityServiceDbContext>
 {
-    protected ILocalEventBus LocalEventBus { get; }
-
     public IdentityServiceDatabaseMigrationChecker(
         IUnitOfWorkManager unitOfWorkManager,
         IServiceProvider serviceProvider,
         ICurrentTenant currentTenant,
         IDistributedEventBus distributedEventBus,
-        ILocalEventBus localEventBus,
         IAbpDistributedLock abpDistributedLock)
         : base(
             unitOfWorkManager,
@@ -29,6 +26,5 @@ public class IdentityServiceDatabaseMigrationChecker : PendingEfCoreMigrationsCh
             abpDistributedLock,
             IdentityServiceDbProperties.ConnectionStringName)
     {
-        LocalEventBus = localEventBus;
     }
 }

services/identity/src/EShopOnAbp.IdentityService.HttpApi.Host/IdentityServiceHttpApiHostModule.cs

@@ -100,6 +100,6 @@ public class IdentityServiceHttpApiHostModule : AbpModule
     {
         await context.ServiceProvider
             .GetRequiredService<IdentityServiceDatabaseMigrationChecker>()
-            .CheckAndApplyDatabaseMigrations();
+            .CheckAndApplyDatabaseMigrationsAsync();
     }
 }

services/ordering/src/EShopOnAbp.OrderingService.HttpApi.Host/OrderingServiceHttpApiHostModule.cs

@@ -109,6 +109,6 @@ public class OrderingServiceHttpApiHostModule : AbpModule
     {
         await context.ServiceProvider
             .GetRequiredService<OrderingServiceDatabaseMigrationChecker>()
-            .CheckAndApplyDatabaseMigrations();
+            .CheckAndApplyDatabaseMigrationsAsync();
     }
 }

services/payment/src/EShopOnAbp.PaymentService.HttpApi.Host/PaymentServiceHttpApiHostModule.cs

@@ -100,6 +100,6 @@ public class PaymentServiceHttpApiHostModule : AbpModule
     {
         await context.ServiceProvider
             .GetRequiredService<PaymentServiceDatabaseMigrationChecker>()
-            .CheckAndApplyDatabaseMigrations();
+            .CheckAndApplyDatabaseMigrationsAsync();
     }
 }

shared/EShopOnAbp.Shared.Hosting.Microservices/DbMigrations/EfCore/PendingEfCoreMigrationsChecker.cs

@@ -9,6 +9,7 @@ using Volo.Abp.DistributedLocking;
 using Volo.Abp.EventBus.Distributed;
 using Volo.Abp.MultiTenancy;
 using Volo.Abp.Uow;
+using Volo.Abp.VirtualFileSystem;
 
 namespace EShopOnAbp.Shared.Hosting.Microservices.DbMigrations.EfCore;
 
@@ -38,7 +39,12 @@ public abstract class PendingEfCoreMigrationsChecker<TDbContext> : PendingMigrat
         DatabaseName = databaseName;
     }
 
-    public virtual async Task CheckAndApplyDatabaseMigrations()
+    public virtual async Task CheckAndApplyDatabaseMigrationsAsync()
+    {
+        await TryAsync(LockAndApplyDatabaseMigrationsAsync); 
+    }
+
+    protected virtual async Task LockAndApplyDatabaseMigrationsAsync()
     {
         await using (var handle = await DistributedLockProvider.TryAcquireAsync("Migration_" + DatabaseName))
         {

shared/EShopOnAbp.Shared.Hosting.Microservices/DbMigrations/MongoDb/PendingMongoDbMigrationsChecker.cs

@@ -39,18 +39,21 @@ public class PendingMongoDbMigrationsChecker<TDbContext> : PendingMigrationsChec
 
     public virtual async Task CheckAndApplyDatabaseMigrationsAsync()
     {
-        using (CurrentTenant.Change(null))
+        await TryAsync(async () =>
         {
-            // Create database tables if needed
+            using (CurrentTenant.Change(null))
-            using (var uow = UnitOfWorkManager.Begin(requiresNew: true, isTransactional: false))
             {
-                await MigrateDatabaseSchemaAsync();
+                // Create database tables if needed
+                using (var uow = UnitOfWorkManager.Begin(requiresNew: true, isTransactional: false))
+                {
+                    await MigrateDatabaseSchemaAsync();
 
-                await DataSeeder.SeedAsync();
+                    await DataSeeder.SeedAsync();
 
-                await uow.CompleteAsync();
+                    await uow.CompleteAsync();
+                }
             }
-        }
+        });
     }
 
     /// <summary>

shared/EShopOnAbp.Shared.Hosting.Microservices/DbMigrations/PendingMigrationsCheckerBase.cs

@@ -1,7 +1,33 @@
-using Volo.Abp.DependencyInjection;
+using Serilog;
+using System;
+using System.Threading.Tasks;
+using Volo.Abp;
+using Volo.Abp.DependencyInjection;
 
 namespace EShopOnAbp.Shared.Hosting.Microservices.DbMigrations;
 
 public abstract class PendingMigrationsCheckerBase : ITransientDependency
 {
+    public async Task TryAsync(Func<Task> task, int retryCount = 3)
+    {
+        try
+        {
+            await task();
+        }
+        catch (Exception ex)
+        {
+            retryCount--;
+
+            if (retryCount <= 0)
+            {
+                throw;
+            }
+
+            Log.Warning($"{ex.GetType().Name} has been thrown. The operation will be tried {retryCount} times more. Exception:\n{ex.Message}");
+
+            await Task.Delay(RandomHelper.GetRandom(5000, 15000));
+
+            await TryAsync(task, retryCount);
+        }
+    }
 }