implement IDynamicWebhookDefinitionStore

aspnet-core/modules/webhooks/LINGYUN.Abp.Webhooks.Core/LINGYUN/Abp/Webhooks/WebhookDefinition.cs

@@ -27,6 +27,11 @@ namespace LINGYUN.Abp.Webhooks
 
         public Dictionary<string, object> Properties { get; }
 
+        public object this[string name] {
+            get => Properties.GetOrDefault(name);
+            set => Properties[name] = value;
+        }
+
         public WebhookDefinition(string name, ILocalizableString displayName = null, ILocalizableString description = null)
         {
             if (name.IsNullOrWhiteSpace())

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain.Shared/LINGYUN/Abp/WebhooksManagement/WebhookDefinitionRecordConsts.cs

@@ -8,7 +8,5 @@ public static class WebhookDefinitionRecordConsts
 
     public static int MaxDescriptionLength { get; set; } = 256;
 
-    public static int MaxProvidersLength { get; set; } = 128;
-
     public static int MaxRequiredFeaturesLength { get; set; } = 256;
 }

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/DynamicWebhookDefinitionStore.cs

@@ -0,0 +1,172 @@
+using LINGYUN.Abp.Webhooks;
+using Microsoft.Extensions.Caching.Distributed;
+using Microsoft.Extensions.Options;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Threading.Tasks;
+using Volo.Abp;
+using Volo.Abp.Caching;
+using Volo.Abp.DependencyInjection;
+using Volo.Abp.DistributedLocking;
+using Volo.Abp.Threading;
+
+namespace LINGYUN.Abp.WebhooksManagement;
+
+[Dependency(ReplaceServices = true)]
+public class DynamicWebhookDefinitionStore : IDynamicWebhookDefinitionStore, ITransientDependency
+{
+    protected IWebhookGroupDefinitionRecordRepository WebhookGroupRepository { get; }
+    protected IWebhookDefinitionRecordRepository WebhookRepository { get; }
+    protected IWebhookDefinitionSerializer WebhookDefinitionSerializer { get; }
+    protected IDynamicWebhookDefinitionStoreCache StoreCache { get; }
+    protected IDistributedCache DistributedCache { get; }
+    protected IAbpDistributedLock DistributedLock { get; }
+    public WebhookManagementOptions WebhookManagementOptions { get; }
+    protected AbpDistributedCacheOptions CacheOptions { get; }
+    
+    public DynamicWebhookDefinitionStore(
+        IWebhookGroupDefinitionRecordRepository webhookGroupRepository,
+        IWebhookDefinitionRecordRepository webhookRepository,
+        IWebhookDefinitionSerializer webhookDefinitionSerializer,
+        IDynamicWebhookDefinitionStoreCache storeCache,
+        IDistributedCache distributedCache,
+        IOptions<AbpDistributedCacheOptions> cacheOptions,
+        IOptions<WebhookManagementOptions> webhookManagementOptions,
+        IAbpDistributedLock distributedLock)
+    {
+        WebhookGroupRepository = webhookGroupRepository;
+        WebhookRepository = webhookRepository;
+        WebhookDefinitionSerializer = webhookDefinitionSerializer;
+        StoreCache = storeCache;
+        DistributedCache = distributedCache;
+        DistributedLock = distributedLock;
+        WebhookManagementOptions = webhookManagementOptions.Value;
+        CacheOptions = cacheOptions.Value;
+    }
+
+    public virtual async Task<WebhookDefinition> GetOrNullAsync(string name)
+    {
+        if (!WebhookManagementOptions.IsDynamicWebhookStoreEnabled)
+        {
+            return null;
+        }
+
+        using (await StoreCache.SyncSemaphore.LockAsync())
+        {
+            await EnsureCacheIsUptoDateAsync();
+            return StoreCache.GetWebhookOrNull(name);
+        }
+    }
+
+    public virtual async Task<IReadOnlyList<WebhookDefinition>> GetWebhooksAsync()
+    {
+        if (!WebhookManagementOptions.IsDynamicWebhookStoreEnabled)
+        {
+            return Array.Empty<WebhookDefinition>();
+        }
+
+        using (await StoreCache.SyncSemaphore.LockAsync())
+        {
+            await EnsureCacheIsUptoDateAsync();
+            return StoreCache.GetWebhooks().ToImmutableList();
+        }
+    }
+
+    public virtual async Task<IReadOnlyList<WebhookGroupDefinition>> GetGroupsAsync()
+    {
+        if (!WebhookManagementOptions.IsDynamicWebhookStoreEnabled)
+        {
+            return Array.Empty<WebhookGroupDefinition>();
+        }
+
+        using (await StoreCache.SyncSemaphore.LockAsync())
+        {
+            await EnsureCacheIsUptoDateAsync();
+            return StoreCache.GetGroups().ToImmutableList();
+        }
+    }
+
+    protected virtual async Task EnsureCacheIsUptoDateAsync()
+    {
+        if (StoreCache.LastCheckTime.HasValue &&
+            DateTime.Now.Subtract(StoreCache.LastCheckTime.Value).TotalSeconds < 30)
+        {
+            /* We get the latest webhook with a small delay for optimization */
+            return;
+        }
+        
+        var stampInDistributedCache = await GetOrSetStampInDistributedCache();
+        
+        if (stampInDistributedCache == StoreCache.CacheStamp)
+        {
+            StoreCache.LastCheckTime = DateTime.Now;
+            return;
+        }
+
+        await UpdateInMemoryStoreCache();
+
+        StoreCache.CacheStamp = stampInDistributedCache;
+        StoreCache.LastCheckTime = DateTime.Now;
+    }
+
+    protected virtual async Task UpdateInMemoryStoreCache()
+    {
+        var webhookGroupRecords = await WebhookGroupRepository.GetListAsync();
+        var webhookRecords = await WebhookRepository.GetListAsync();
+
+        await StoreCache.FillAsync(webhookGroupRecords, webhookRecords);
+    }
+
+    protected virtual async Task<string> GetOrSetStampInDistributedCache()
+    {
+        var cacheKey = GetCommonStampCacheKey();
+
+        var stampInDistributedCache = await DistributedCache.GetStringAsync(cacheKey);
+        if (stampInDistributedCache != null)
+        {
+            return stampInDistributedCache;
+        }
+
+        await using (var commonLockHandle = await DistributedLock
+                         .TryAcquireAsync(GetCommonDistributedLockKey(), TimeSpan.FromMinutes(2)))
+        {
+            if (commonLockHandle == null)
+            {
+                /* This request will fail */
+                throw new AbpException(
+                    "Could not acquire distributed lock for webhook definition common stamp check!"
+                );
+            }
+
+            stampInDistributedCache = await DistributedCache.GetStringAsync(cacheKey);
+            if (stampInDistributedCache != null)
+            {
+                return stampInDistributedCache;
+            }
+
+            stampInDistributedCache = Guid.NewGuid().ToString();
+            
+            await DistributedCache.SetStringAsync(
+                cacheKey,
+                stampInDistributedCache,
+                new DistributedCacheEntryOptions
+                {
+                    SlidingExpiration = TimeSpan.FromDays(30) //TODO: Make it configurable?
+                }
+            );
+        }
+
+        return stampInDistributedCache;
+    }
+
+    protected virtual string GetCommonStampCacheKey()
+    {
+        return $"{CacheOptions.KeyPrefix}_AbpInMemoryWebhookCacheStamp";
+    }
+    
+    protected virtual string GetCommonDistributedLockKey()
+    {
+        return $"{CacheOptions.KeyPrefix}_Common_AbpWebhookUpdateLock";
+    }
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/DynamicWebhookDefinitionStoreInMemoryCache.cs

@@ -0,0 +1,114 @@
+using LINGYUN.Abp.Webhooks;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Volo.Abp.DependencyInjection;
+using Volo.Abp.Localization;
+using Volo.Abp.SimpleStateChecking;
+
+namespace LINGYUN.Abp.WebhooksManagement;
+
+[ExposeServices(
+    typeof(IDynamicWebhookDefinitionStoreCache),
+    typeof(DynamicWebhookDefinitionStoreInMemoryCache))]
+public class DynamicWebhookDefinitionStoreInMemoryCache :
+    IDynamicWebhookDefinitionStoreCache,
+    ISingletonDependency
+{
+    public string CacheStamp { get; set; }
+    
+    protected IDictionary<string, WebhookGroupDefinition> WebhookGroupDefinitions { get; }
+    protected IDictionary<string, WebhookDefinition> WebhookDefinitions { get; }
+    protected ISimpleStateCheckerSerializer StateCheckerSerializer { get; }
+    protected ILocalizableStringSerializer LocalizableStringSerializer { get; }
+
+    public SemaphoreSlim SyncSemaphore { get; } = new(1, 1);
+    
+    public DateTime? LastCheckTime { get; set; }
+
+    public DynamicWebhookDefinitionStoreInMemoryCache(
+        ISimpleStateCheckerSerializer stateCheckerSerializer, 
+        ILocalizableStringSerializer localizableStringSerializer)
+    {
+        StateCheckerSerializer = stateCheckerSerializer;
+        LocalizableStringSerializer = localizableStringSerializer;
+        
+        WebhookGroupDefinitions = new Dictionary<string, WebhookGroupDefinition>();
+        WebhookDefinitions = new Dictionary<string, WebhookDefinition>();
+    }
+
+    public Task FillAsync(
+        List<WebhookGroupDefinitionRecord> webhookGroupRecords,
+        List<WebhookDefinitionRecord> webhookRecords)
+    {
+        WebhookGroupDefinitions.Clear();
+        WebhookDefinitions.Clear();
+        
+        var context = new WebhookDefinitionContext(null);
+        
+        foreach (var webhookGroupRecord in webhookGroupRecords)
+        {
+            var webhookGroup = context.AddGroup(
+                webhookGroupRecord.Name,
+                LocalizableStringSerializer.Deserialize(webhookGroupRecord.DisplayName)
+            );
+            
+            WebhookGroupDefinitions[webhookGroup.Name] = webhookGroup;
+
+            foreach (var property in webhookGroupRecord.ExtraProperties)
+            {
+                webhookGroup[property.Key] = property.Value;
+            }
+
+            var webhookRecordsInThisGroup = webhookRecords
+                .Where(p => p.GroupName == webhookGroup.Name);
+
+            foreach (var webhookRecord in webhookRecordsInThisGroup)
+            {
+                AddWebhook(webhookGroup, webhookRecord);
+            }
+        }
+
+        return Task.CompletedTask;
+    }
+
+    public WebhookDefinition GetWebhookOrNull(string name)
+    {
+        return WebhookDefinitions.GetOrDefault(name);
+    }
+
+    public IReadOnlyList<WebhookDefinition> GetWebhooks()
+    {
+        return WebhookDefinitions.Values.ToList();
+    }
+
+    public IReadOnlyList<WebhookGroupDefinition> GetGroups()
+    {
+        return WebhookGroupDefinitions.Values.ToList();
+    }
+
+    private void AddWebhook(
+        WebhookGroupDefinition webhookGroup,
+        WebhookDefinitionRecord webhookRecord)
+    {
+        var webhook = webhookGroup.AddWebhook(
+            webhookRecord.Name,
+            LocalizableStringSerializer.Deserialize(webhookRecord.DisplayName),
+            LocalizableStringSerializer.Deserialize(webhookRecord.Description)
+        );
+        
+        WebhookDefinitions[webhook.Name] = webhook;
+
+        if (!webhookRecord.RequiredFeatures.IsNullOrWhiteSpace())
+        {
+            webhook.RequiredFeatures.AddRange(webhookRecord.RequiredFeatures.Split(','));
+        }
+
+        foreach (var property in webhookRecord.ExtraProperties)
+        {
+            webhook[property.Key] = property.Value;
+        }
+    }
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/IDynamicWebhookDefinitionStoreCache.cs

@@ -0,0 +1,26 @@
+using LINGYUN.Abp.Webhooks;
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace LINGYUN.Abp.WebhooksManagement;
+
+public interface IDynamicWebhookDefinitionStoreCache
+{
+    string CacheStamp { get; set; }
+    
+    SemaphoreSlim SyncSemaphore { get; }
+    
+    DateTime? LastCheckTime { get; set; }
+
+    Task FillAsync(
+        List<WebhookGroupDefinitionRecord> webhookGroupRecords,
+        List<WebhookDefinitionRecord> webhookRecords);
+
+    WebhookDefinition GetWebhookOrNull(string name);
+    
+    IReadOnlyList<WebhookDefinition> GetWebhooks();
+    
+    IReadOnlyList<WebhookGroupDefinition> GetGroups();
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/IWebhookDefinitionSerializer.cs

@@ -0,0 +1,19 @@
+using JetBrains.Annotations;
+using LINGYUN.Abp.Webhooks;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace LINGYUN.Abp.WebhooksManagement;
+
+public interface IWebhookDefinitionSerializer
+{
+    Task<(WebhookGroupDefinitionRecord[], WebhookDefinitionRecord[])>
+        SerializeAsync(IEnumerable<WebhookGroupDefinition> WebhookGroups);
+
+    Task<WebhookGroupDefinitionRecord> SerializeAsync(
+        WebhookGroupDefinition WebhookGroup);
+
+    Task<WebhookDefinitionRecord> SerializeAsync(
+        WebhookDefinition Webhook,
+        [CanBeNull] WebhookGroupDefinition WebhookGroup);
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/WebhookDefinitionRecord.cs

@@ -21,8 +21,6 @@ public class WebhookDefinitionRecord : BasicAggregateRoot<Guid>, IHasExtraProper
 
     public bool IsEnabled { get; set; }
 
-    public string Providers { get; set; }
-
     public string RequiredFeatures { get; set; }
 
     public ExtraPropertyDictionary ExtraProperties { get; protected set; }
@@ -40,7 +38,6 @@ public class WebhookDefinitionRecord : BasicAggregateRoot<Guid>, IHasExtraProper
         string displayName,
         string description = null,
         bool isEnabled = true,
-        string providers = null,
         string requiredFeatures = null)
         : base(id)
     {
@@ -48,8 +45,6 @@ public class WebhookDefinitionRecord : BasicAggregateRoot<Guid>, IHasExtraProper
         Name = Check.NotNullOrWhiteSpace(name, nameof(name), WebhookDefinitionRecordConsts.MaxNameLength);
         DisplayName = Check.NotNullOrWhiteSpace(displayName, nameof(displayName), WebhookDefinitionRecordConsts.MaxDisplayNameLength);
         Description = Check.Length(description, nameof(description), WebhookDefinitionRecordConsts.MaxDescriptionLength);
-
-        Providers = Check.Length(providers, nameof(providers), WebhookDefinitionRecordConsts.MaxProvidersLength);
         RequiredFeatures = Check.Length(requiredFeatures, nameof(requiredFeatures), WebhookDefinitionRecordConsts.MaxRequiredFeaturesLength);
 
         IsEnabled = isEnabled;
@@ -85,11 +80,6 @@ public class WebhookDefinitionRecord : BasicAggregateRoot<Guid>, IHasExtraProper
             return false;
         }
 
-        if (Providers != otherRecord.Providers)
-        {
-            return false;
-        }
-
         if (RequiredFeatures != otherRecord.RequiredFeatures)
         {
             return false;
@@ -130,11 +120,6 @@ public class WebhookDefinitionRecord : BasicAggregateRoot<Guid>, IHasExtraProper
             IsEnabled = otherRecord.IsEnabled;
         }
 
-        if (Providers != otherRecord.Providers)
-        {
-            Providers = otherRecord.Providers;
-        }
-
         if (RequiredFeatures != otherRecord.RequiredFeatures)
         {
             RequiredFeatures = otherRecord.RequiredFeatures;

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/WebhookDefinitionSerializer.cs

@@ -0,0 +1,99 @@
+using LINGYUN.Abp.Webhooks;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Threading.Tasks;
+using Volo.Abp.Data;
+using Volo.Abp.DependencyInjection;
+using Volo.Abp.Guids;
+using Volo.Abp.Localization;
+using Volo.Abp.SimpleStateChecking;
+
+namespace LINGYUN.Abp.WebhooksManagement;
+
+public class WebhookDefinitionSerializer : IWebhookDefinitionSerializer, ITransientDependency
+{
+    protected ISimpleStateCheckerSerializer StateCheckerSerializer { get; }
+    protected IGuidGenerator GuidGenerator { get; }
+    protected ILocalizableStringSerializer LocalizableStringSerializer { get; }
+
+    public WebhookDefinitionSerializer(
+        IGuidGenerator guidGenerator,
+        ISimpleStateCheckerSerializer stateCheckerSerializer, 
+        ILocalizableStringSerializer localizableStringSerializer)
+    {
+        StateCheckerSerializer = stateCheckerSerializer;
+        LocalizableStringSerializer = localizableStringSerializer;
+        GuidGenerator = guidGenerator;
+    }
+
+    public async Task<(WebhookGroupDefinitionRecord[], WebhookDefinitionRecord[])> 
+        SerializeAsync(IEnumerable<WebhookGroupDefinition> webhookGroups)
+    {
+        var webhookGroupRecords = new List<WebhookGroupDefinitionRecord>();
+        var webhookRecords = new List<WebhookDefinitionRecord>();
+
+        foreach (var webhookGroup in webhookGroups)
+        {
+            webhookGroupRecords.Add(await SerializeAsync(webhookGroup));
+            
+            foreach (var webhook in webhookGroup.Webhooks)
+            {
+                webhookRecords.Add(await SerializeAsync(webhook, webhookGroup));
+            }
+        }
+
+        return (webhookGroupRecords.ToArray(), webhookRecords.ToArray());
+    }
+    
+    public Task<WebhookGroupDefinitionRecord> SerializeAsync(WebhookGroupDefinition webhookGroup)
+    {
+        using (CultureHelper.Use(CultureInfo.InvariantCulture))
+        {
+            var webhookGroupRecord = new WebhookGroupDefinitionRecord(
+                GuidGenerator.Create(),
+                webhookGroup.Name,
+                LocalizableStringSerializer.Serialize(webhookGroup.DisplayName)
+            );
+
+            foreach (var property in webhookGroup.Properties)
+            {
+                webhookGroupRecord.SetProperty(property.Key, property.Value);
+            }
+            
+            return Task.FromResult(webhookGroupRecord);
+        }
+    }
+    
+    public Task<WebhookDefinitionRecord> SerializeAsync(
+        WebhookDefinition webhook,
+        WebhookGroupDefinition webhookGroup)
+    {
+        using (CultureHelper.Use(CultureInfo.InvariantCulture))
+        {
+            var webhookRecord = new WebhookDefinitionRecord(
+                GuidGenerator.Create(),
+                webhookGroup?.Name,
+                webhook.Name,
+                LocalizableStringSerializer.Serialize(webhook.DisplayName),
+                LocalizableStringSerializer.Serialize(webhook.Description),
+                true,
+                SerializeRequiredFeatures(webhook.RequiredFeatures)
+            );
+
+            foreach (var property in webhook.Properties)
+            {
+                webhookRecord.SetProperty(property.Key, property.Value);
+            }
+            
+            return Task.FromResult(webhookRecord);
+        }
+    }
+    
+    protected virtual string SerializeRequiredFeatures(List<string> requiredFeatures)
+    {
+        return requiredFeatures.Any()
+            ? requiredFeatures.JoinAsString(",")
+            : null;
+    }
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.Domain/LINGYUN/Abp/WebhooksManagement/WebhookManagementOptions.cs

@@ -0,0 +1,10 @@
+namespace LINGYUN.Abp.WebhooksManagement;
+public class WebhookManagementOptions
+{
+    public bool IsDynamicWebhookStoreEnabled { get; set; }
+
+    public WebhookManagementOptions()
+    {
+        IsDynamicWebhookStoreEnabled = true;
+    }
+}

aspnet-core/modules/webhooks/LINGYUN.Abp.WebhooksManagement.EntityFrameworkCore/LINGYUN/Abp/WebhooksManagement/EntityFrameworkCore/WebhooksManagementDbContextModelCreatingExtensions.cs

@@ -105,7 +105,6 @@ public static class WebhooksManagementDbContextModelCreatingExtensions
             b.Property(x => x.Name).HasMaxLength(WebhookDefinitionRecordConsts.MaxNameLength).IsRequired();
             b.Property(x => x.DisplayName).HasMaxLength(WebhookDefinitionRecordConsts.MaxDisplayNameLength).IsRequired();
             b.Property(x => x.Description).HasMaxLength(WebhookDefinitionRecordConsts.MaxDescriptionLength);
-            b.Property(x => x.Providers).HasMaxLength(WebhookDefinitionRecordConsts.MaxProvidersLength);
             b.Property(x => x.RequiredFeatures).HasMaxLength(WebhookDefinitionRecordConsts.MaxRequiredFeaturesLength);
 
             b.HasIndex(x => new { x.Name }).IsUnique();