Notifications deduplication; default exceeded rate limits notification rules

application/src/main/java/org/thingsboard/server/service/notification/rule/DefaultNotificationRuleProcessor.java

@@ -15,7 +15,6 @@
  */
 package org.thingsboard.server.service.notification.rule;
 
-import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,6 +30,7 @@ import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.NotificationRequestId;
 import org.thingsboard.server.common.data.id.NotificationRuleId;
 import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.common.data.notification.NotificationRequest;
 import org.thingsboard.server.common.data.notification.NotificationRequestConfig;
 import org.thingsboard.server.common.data.notification.NotificationRequestStatus;
@@ -39,22 +39,20 @@ import org.thingsboard.server.common.data.notification.rule.NotificationRule;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerType;
 import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
+import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
 import org.thingsboard.server.common.msg.notification.trigger.NotificationRuleTrigger;
 import org.thingsboard.server.common.msg.notification.trigger.RuleEngineMsgTrigger;
 import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
 import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.dao.notification.NotificationRequestService;
-import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.dao.util.limits.RateLimitService;
 import org.thingsboard.server.queue.discovery.PartitionService;
-import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
 import org.thingsboard.server.service.executors.NotificationExecutorService;
 import org.thingsboard.server.service.notification.rule.cache.NotificationRulesCache;
 import org.thingsboard.server.service.notification.rule.trigger.NotificationRuleTriggerProcessor;
 import org.thingsboard.server.service.notification.rule.trigger.RuleEngineMsgNotificationRuleTriggerProcessor;
 
 import javax.annotation.PostConstruct;
-import java.io.Serializable;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -62,6 +60,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 @Service
@@ -142,7 +141,7 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess
                 log.debug("[{}] Rate limit for notification requests per rule was exceeded (rule '{}')", rule.getTenantId(), rule.getName());
                 return;
             }
-            if (trigger.getType().isDeduplicate() && alreadySent(rule.getId(), trigger)) {
+            if (trigger.deduplicate() && alreadySent(rule, trigger)) {
                 return;
             }
 
@@ -194,21 +193,29 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess
         return triggerProcessors.get(triggerConfig.getTriggerType()).constructNotificationInfo(trigger);
     }
 
-    private boolean alreadySent(NotificationRuleId ruleId, NotificationRuleTrigger trigger) {
-        String key = ruleId + "_" + trigger.getOriginatorEntityId();
-        SentNotification sent = sentNotifications.get(key, SentNotification.class);
-        boolean alreadySent;
-        if (sent != null && sent.getTrigger().equals(trigger)) {
-            alreadySent = true;
-            log.debug("Notification for {} trigger was already sent, ignoring", trigger.getType());
-            // updating cache anyway so that the value is not removed by ttl
+    private boolean alreadySent(NotificationRule rule, NotificationRuleTrigger trigger) {
+        String deduplicationKey = String.join("_", rule.getDeduplicationKey(), trigger.getDeduplicationKey());
+
+        AtomicBoolean isNew = new AtomicBoolean(false);
+        Long lastSentTs = sentNotifications.get(deduplicationKey, () -> {
+            isNew.set(true);
+            log.trace("[{}] Putting to sentNotifications cache: {}", rule.getId(), trigger);
+            return System.currentTimeMillis();
+        });
+        if (isNew.get()) {
+            return false;
+        }
+
+        long deduplicationDuration = trigger.getDeduplicationDuration();
+        long passed = System.currentTimeMillis() - lastSentTs;
+        if (deduplicationDuration == 0 || passed <= deduplicationDuration) {
+            log.debug("Notification for {} trigger was already sent {} ms ago, deduplication duration is {} ms. Key: '{}'",
+                    trigger.getType(), passed, deduplicationDuration, deduplicationKey);
+            sentNotifications.put(deduplicationKey, lastSentTs); // updating cache anyway so that the value is not removed by ttl
+            return true;
         } else {
-            alreadySent = false;
-            sent = new SentNotification(trigger);
+            return false;
         }
-        log.trace("[{}] Putting to sentNotifications cache: {}", ruleId, trigger);
-        sentNotifications.put(key, sent);
-        return alreadySent;
     }
 
     @EventListener(ComponentLifecycleMsg.class)
@@ -245,11 +252,4 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess
         RuleEngineMsgTrigger.msgTypeToTriggerType = ruleEngineMsgTypeToTriggerType;
     }
 
-    @Data
-    private static class SentNotification implements Serializable {
-        private static final long serialVersionUID = 38973480405095422L;
-
-        private final NotificationRuleTrigger trigger;
-    }
-
 }

application/src/main/java/org/thingsboard/server/service/notification/rule/cache/DefaultNotificationRulesCache.java

@@ -17,7 +17,6 @@ package org.thingsboard.server.service.notification.rule.cache;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
-import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
@@ -50,7 +49,7 @@ public class DefaultNotificationRulesCache implements NotificationRulesCache {
     private int cacheMaxSize;
     @Value("${cache.notificationRules.timeToLiveInMinutes:30}")
     private int cacheValueTtl;
-    private Cache<CacheKey, List<NotificationRule>> cache;
+    private Cache<String, List<NotificationRule>> cache;
 
     private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
@@ -96,21 +95,15 @@ public class DefaultNotificationRulesCache implements NotificationRulesCache {
         }
     }
 
-    private void evict(TenantId tenantId) {
+    public void evict(TenantId tenantId) {
         cache.invalidateAll(Arrays.stream(NotificationRuleTriggerType.values())
                 .map(triggerType -> key(tenantId, triggerType))
                 .collect(Collectors.toList()));
         log.trace("Evicted all notification rules for tenant {} from cache", tenantId);
     }
 
-    private static CacheKey key(TenantId tenantId, NotificationRuleTriggerType triggerType) {
-        return new CacheKey(tenantId, triggerType);
-    }
-
-    @Data
-    private static class CacheKey {
-        private final TenantId tenantId;
-        private final NotificationRuleTriggerType triggerType;
+    private static String key(TenantId tenantId, NotificationRuleTriggerType triggerType) {
+        return tenantId + "_" + triggerType;
     }
 
 }

application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RateLimitsTriggerProcessor.java

@@ -2,18 +2,20 @@ package org.thingsboard.server.service.notification.rule.trigger;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.notification.info.RateLimitsNotificationInfo;
 import org.thingsboard.server.common.data.notification.info.RuleOriginatedNotificationInfo;
-import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerType;
 import org.thingsboard.server.common.data.notification.rule.trigger.RateLimitsNotificationRuleTriggerConfig;
-import org.thingsboard.server.common.msg.notification.trigger.NotificationRuleTrigger;
 import org.thingsboard.server.common.msg.notification.trigger.RateLimitsTrigger;
+import org.thingsboard.server.dao.tenant.TenantService;
 
 @Service
 @RequiredArgsConstructor
 public class RateLimitsTriggerProcessor implements NotificationRuleTriggerProcessor<RateLimitsTrigger, RateLimitsNotificationRuleTriggerConfig> {
 
+    private final TenantService tenantService;
+
     @Override
     public boolean matchesFilter(RateLimitsTrigger trigger, RateLimitsNotificationRuleTriggerConfig triggerConfig) {
         return trigger.getLimitLevel() != null;
@@ -23,7 +25,10 @@ public class RateLimitsTriggerProcessor implements NotificationRuleTriggerProces
     public RuleOriginatedNotificationInfo constructNotificationInfo(RateLimitsTrigger trigger) {
         return RateLimitsNotificationInfo.builder()
                 .tenantId(trigger.getTenantId())
+                .tenantName(tenantService.findTenantById(trigger.getTenantId()).getName())
                 .api(trigger.getApi())
+                .limitLevel(trigger.getLimitLevel() instanceof TenantId ? null : trigger.getLimitLevel())
+                .limitLevelEntityName(trigger.getLimitLevelEntityName())
                 .build();
     }
 

application/src/test/java/org/thingsboard/server/service/limits/RateLimitServiceTest.java

@@ -24,11 +24,12 @@ import org.thingsboard.server.common.data.TenantProfile;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.NotificationRuleId;
 import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
 import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
+import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
 import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
 import org.thingsboard.server.dao.util.limits.DefaultRateLimitService;
-import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.dao.util.limits.RateLimitService;
 
 import java.util.List;
@@ -37,6 +38,7 @@ import java.util.UUID;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.when;
 
@@ -50,7 +52,7 @@ public class RateLimitServiceTest {
     @Before
     public void beforeEach() {
         tenantProfileCache = Mockito.mock(TbTenantProfileCache.class);
-        rateLimitService = new DefaultRateLimitService(tenantProfileCache, 60, 100);
+        rateLimitService = new DefaultRateLimitService(tenantProfileCache, mock(NotificationRuleProcessor.class), 60, 100);
         tenantId = new TenantId(UUID.randomUUID());
     }
 

application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java

@@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.notification.settings.NotificationSett
 import org.thingsboard.server.common.data.notification.targets.NotificationTarget;
 import org.thingsboard.server.common.data.notification.targets.platform.PlatformUsersNotificationTargetConfig;
 import org.thingsboard.server.common.data.notification.targets.platform.UserListFilter;
+import org.thingsboard.server.common.data.notification.targets.platform.UsersFilter;
 import org.thingsboard.server.common.data.notification.template.DeliveryMethodNotificationTemplate;
 import org.thingsboard.server.common.data.notification.template.EmailDeliveryMethodNotificationTemplate;
 import org.thingsboard.server.common.data.notification.template.NotificationTemplate;
@@ -80,13 +81,16 @@ public abstract class AbstractNotificationApiTest extends AbstractControllerTest
     public static final NotificationType DEFAULT_NOTIFICATION_TYPE = NotificationType.GENERAL;
 
     protected NotificationTarget createNotificationTarget(UserId... usersIds) {
-        NotificationTarget notificationTarget = new NotificationTarget();
-        notificationTarget.setTenantId(tenantId);
-        notificationTarget.setName("Users " + List.of(usersIds));
-        PlatformUsersNotificationTargetConfig targetConfig = new PlatformUsersNotificationTargetConfig();
         UserListFilter filter = new UserListFilter();
         filter.setUsersIds(DaoUtil.toUUIDs(List.of(usersIds)));
-        targetConfig.setUsersFilter(filter);
+        return createNotificationTarget(filter);
+    }
+
+    protected NotificationTarget createNotificationTarget(UsersFilter usersFilter) {
+        NotificationTarget notificationTarget = new NotificationTarget();
+        notificationTarget.setName(usersFilter.toString());
+        PlatformUsersNotificationTargetConfig targetConfig = new PlatformUsersNotificationTargetConfig();
+        targetConfig.setUsersFilter(usersFilter);
         notificationTarget.setConfiguration(targetConfig);
         return saveNotificationTarget(notificationTarget);
     }

application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java

@@ -23,6 +23,7 @@ import org.junit.function.ThrowingRunnable;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.SpyBean;
 import org.springframework.data.util.Pair;
+import org.springframework.test.context.TestPropertySource;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.Device;
@@ -42,6 +43,9 @@ import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType;
 import org.thingsboard.server.common.data.device.profile.AlarmRule;
 import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm;
 import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpec;
+import org.thingsboard.server.common.data.id.NotificationRuleId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.common.data.notification.Notification;
 import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod;
 import org.thingsboard.server.common.data.notification.NotificationRequest;
@@ -59,6 +63,8 @@ import org.thingsboard.server.common.data.notification.rule.trigger.EntityAction
 import org.thingsboard.server.common.data.notification.rule.trigger.NewPlatformVersionNotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerType;
 import org.thingsboard.server.common.data.notification.targets.NotificationTarget;
+import org.thingsboard.server.common.data.notification.targets.platform.AffectedTenantAdministratorsFilter;
+import org.thingsboard.server.common.data.notification.targets.platform.SystemAdministratorsFilter;
 import org.thingsboard.server.common.data.notification.template.NotificationTemplate;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
@@ -68,13 +74,14 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.security.Authority;
+import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
 import org.thingsboard.server.common.msg.notification.trigger.NewPlatformVersionTrigger;
+import org.thingsboard.server.dao.notification.DefaultNotifications;
 import org.thingsboard.server.dao.notification.NotificationRequestService;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.dao.service.DaoSqlTest;
-import org.thingsboard.server.common.data.limit.LimitedApi;
 import org.thingsboard.server.dao.util.limits.RateLimitService;
-import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
+import org.thingsboard.server.service.notification.rule.cache.DefaultNotificationRulesCache;
 import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
 
 import java.util.ArrayList;
@@ -96,6 +103,9 @@ import static org.awaitility.Awaitility.await;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @DaoSqlTest
+@TestPropertySource(properties = {
+        "transport.http.enabled=true",
+})
 public class NotificationRuleApiTest extends AbstractNotificationApiTest {
 
     @SpyBean
@@ -108,6 +118,10 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest {
     private RuleChainService ruleChainService;
     @Autowired
     private NotificationRuleProcessor notificationRuleProcessor;
+    @Autowired
+    private DefaultNotifications defaultNotifications;
+    @Autowired
+    private DefaultNotificationRulesCache notificationRulesCache;
 
     @Before
     public void beforeEach() throws Exception {
@@ -370,6 +384,62 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest {
         });
     }
 
+    @Test
+    public void testNotificationRuleProcessing_exceededRateLimits() throws Exception {
+        loginSysAdmin();
+        NotificationTarget sysadmins = createNotificationTarget(new SystemAdministratorsFilter());
+        NotificationTarget affectedTenantAdmins = createNotificationTarget(new AffectedTenantAdministratorsFilter());
+        defaultNotifications.create(TenantId.SYS_TENANT_ID, DefaultNotifications.exceededRateLimitsForSysadmin, sysadmins.getId());
+        defaultNotifications.create(TenantId.SYS_TENANT_ID, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId());
+        notificationRulesCache.evict(TenantId.SYS_TENANT_ID);
+
+        int n = 10;
+        updateDefaultTenantProfile(profileConfiguration -> {
+            profileConfiguration.setTenantEntityExportRateLimit(n + ":600");
+            profileConfiguration.setCustomerServerRestLimitsConfiguration(n + ":600");
+            profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(n + ":600");
+            profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":600");
+        });
+        loginTenantAdmin();
+        UUID uuid = UUID.randomUUID();
+        Device device = createDevice("exceeded", "1234");
+        for (int i = 1; i <= n * 2; i++) {
+            rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, tenantId);
+            rateLimitService.checkRateLimit(LimitedApi.REST_REQUESTS, tenantId, customerId);
+            rateLimitService.checkRateLimit(LimitedApi.NOTIFICATION_REQUESTS_PER_RULE, tenantId, new NotificationRuleId(uuid));
+            doPost("/api/v1/1234/telemetry", "{\"dp1\":123}", String.class);
+        }
+
+        loginTenantAdmin();
+        List<Notification> notifications = await().atMost(30, TimeUnit.SECONDS)
+                .until(() -> getMyNotifications(true, 10), list -> list.size() == 4);
+        assertThat(notifications).allSatisfy(notification -> {
+            assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded");
+        });
+        assertThat(notifications).anySatisfy(notification -> {
+            assertThat(notification.getText()).isEqualTo("Rate limits for entity version creation exceeded");
+        });
+        assertThat(notifications).anySatisfy(notification -> {
+            assertThat(notification.getText()).isEqualTo("Rate limits for REST API requests per customer " +
+                    "exceeded for customer " + customerId);
+        });
+        assertThat(notifications).anySatisfy(notification -> {
+            assertThat(notification.getText()).isEqualTo("Rate limits for notification requests " +
+                    "per notification rule exceeded for notification rule " + uuid);
+        });
+        assertThat(notifications).anySatisfy(notification -> {
+            assertThat(notification.getText()).isEqualTo("Rate limits for transport messages " +
+                    "per device exceeded for device '" + device.getName() + "'");
+        });
+
+        loginSysAdmin();
+        notifications = await().atMost(30, TimeUnit.SECONDS)
+                .until(() -> getMyNotifications(true, 10), list -> list.size() == 4);
+        assertThat(notifications).allSatisfy(notification -> {
+            assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME);
+        });
+    }
+
     @Test
     public void testNotificationRuleInfo() throws Exception {
         NotificationDeliveryMethod[] deliveryMethods = {NotificationDeliveryMethod.WEB, NotificationDeliveryMethod.EMAIL};

common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java

@@ -28,10 +28,10 @@ public enum LimitedApi {
     ENTITY_EXPORT(DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit, "entity version creation"),
     ENTITY_IMPORT(DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit, "entity version load"),
     NOTIFICATION_REQUESTS(DefaultTenantProfileConfiguration::getTenantNotificationRequestsRateLimit, "notification requests"),
-    NOTIFICATION_REQUESTS_PER_RULE(DefaultTenantProfileConfiguration::getTenantNotificationRequestsPerRuleRateLimit, "notification requests per rule"),
+    NOTIFICATION_REQUESTS_PER_RULE(DefaultTenantProfileConfiguration::getTenantNotificationRequestsPerRuleRateLimit, "notification requests"),
     REST_REQUESTS((profileConfiguration, level) -> ((EntityId) level).getEntityType() == EntityType.TENANT ?
             profileConfiguration.getTenantServerRestLimitsConfiguration() :
-            profileConfiguration.getCustomerServerRestLimitsConfiguration(), "REST API requests"), // FIXME: different labels depending on level
+            profileConfiguration.getCustomerServerRestLimitsConfiguration(), "REST API requests"),
     WS_UPDATES_PER_SESSION(DefaultTenantProfileConfiguration::getWsUpdatesPerSessionRateLimit, "WS updates per session"),
     CASSANDRA_QUERIES(DefaultTenantProfileConfiguration::getCassandraQueryTenantRateLimitsConfiguration, "Cassandra queries"),
     PASSWORD_RESET(true),

common/data/src/main/java/org/thingsboard/server/common/data/notification/info/RateLimitsNotificationInfo.java

@@ -4,6 +4,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 
 import java.util.Map;
@@ -17,18 +18,26 @@ import static org.thingsboard.server.common.data.util.CollectionsUtil.mapOf;
 public class RateLimitsNotificationInfo implements RuleOriginatedNotificationInfo {
 
     private TenantId tenantId;
+    private String tenantName;
     private String api;
+    private EntityId limitLevel;
+    private String limitLevelEntityName;
 
     @Override
     public Map<String, String> getTemplateData() {
         return mapOf(
-            "api", api
+                "api", api,
+                "limitLevelEntityType", limitLevel != null ? limitLevel.getEntityType().getNormalName() : null,
+                "limitLevelEntityId", limitLevel != null ? limitLevel.getId().toString() : null,
+                "limitLevelEntityName", limitLevelEntityName,
+                "tenantName", tenantName,
+                "tenantId", tenantId.toString()
         );
     }
 
     @Override
     public TenantId getAffectedTenantId() {
-        return RuleOriginatedNotificationInfo.super.getAffectedTenantId();
+        return tenantId;
     }
 
 }

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRule.java

@@ -35,6 +35,8 @@ import javax.validation.constraints.AssertTrue;
 import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.NotNull;
 import java.io.Serializable;
+import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @NoArgsConstructor
@@ -80,4 +82,12 @@ public class NotificationRule extends BaseData<NotificationRuleId> implements Ha
                 triggerType == recipientsConfig.getTriggerType();
     }
 
+    @JsonIgnore
+    public String getDeduplicationKey() {
+        String targets = recipientsConfig.getTargetsTable().values().stream()
+                .flatMap(List::stream).sorted().map(Object::toString)
+                .collect(Collectors.joining(","));
+        return String.join(":", triggerType.toString(), targets, triggerConfig.getDeduplicationKey());
+    }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTriggerConfig.java

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.common.data.notification.rule.trigger;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
@@ -34,9 +35,15 @@ import java.io.Serializable;
         @Type(value = NewPlatformVersionNotificationRuleTriggerConfig.class, name = "NEW_PLATFORM_VERSION"),
         @Type(value = EntitiesLimitNotificationRuleTriggerConfig.class, name = "ENTITIES_LIMIT"),
         @Type(value = ApiUsageLimitNotificationRuleTriggerConfig.class, name = "API_USAGE_LIMIT"),
+        @Type(value = RateLimitsNotificationRuleTriggerConfig.class, name = "RATE_LIMITS"),
 })
 public interface NotificationRuleTriggerConfig extends Serializable {
 
     NotificationRuleTriggerType getTriggerType();
 
+    @JsonIgnore
+    default String getDeduplicationKey() {
+        return "#";
+    }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTriggerType.java

@@ -16,10 +16,8 @@
 package org.thingsboard.server.common.data.notification.rule.trigger;
 
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 
 @Getter
-@RequiredArgsConstructor
 public enum NotificationRuleTriggerType {
 
     ENTITY_ACTION,
@@ -28,16 +26,19 @@ public enum NotificationRuleTriggerType {
     ALARM_ASSIGNMENT,
     DEVICE_ACTIVITY,
     RULE_ENGINE_COMPONENT_LIFECYCLE_EVENT,
-    NEW_PLATFORM_VERSION(false, true),
-    ENTITIES_LIMIT(false, false),
-    API_USAGE_LIMIT(false, false),
-    RATE_LIMITS(false, false);
+    NEW_PLATFORM_VERSION(false),
+    ENTITIES_LIMIT(false),
+    API_USAGE_LIMIT(false),
+    RATE_LIMITS(false);
 
     private final boolean tenantLevel;
-    private final boolean deduplicate;
 
     NotificationRuleTriggerType() {
-        this(true, false);
+        this(true);
+    }
+
+    NotificationRuleTriggerType(boolean tenantLevel) {
+        this.tenantLevel = tenantLevel;
     }
 
 }

common/message/src/main/java/org/thingsboard/server/common/msg/notification/trigger/NewPlatformVersionTrigger.java

@@ -43,4 +43,20 @@ public class NewPlatformVersionTrigger implements NotificationRuleTrigger {
         return TenantId.SYS_TENANT_ID;
     }
 
+
+    @Override
+    public boolean deduplicate() {
+        return true;
+    }
+
+    @Override
+    public String getDeduplicationKey() {
+        return String.join(":", updateInfo.getCurrentVersion(), updateInfo.getLatestVersion());
+    }
+
+    @Override
+    public long getDeduplicationDuration() {
+        return 0;
+    }
+
 }

common/message/src/main/java/org/thingsboard/server/common/msg/notification/trigger/NotificationRuleTrigger.java

@@ -29,4 +29,18 @@ public interface NotificationRuleTrigger extends Serializable {
 
     EntityId getOriginatorEntityId();
 
+
+    default boolean deduplicate() {
+        return false;
+    }
+
+    default String getDeduplicationKey() {
+        EntityId originatorEntityId = getOriginatorEntityId();
+        return String.join(":", originatorEntityId.getEntityType().toString(), originatorEntityId.getId().toString());
+    }
+
+    default long getDeduplicationDuration() {
+        return 0;
+    }
+
 }

common/message/src/main/java/org/thingsboard/server/common/msg/notification/trigger/RateLimitsTrigger.java

@@ -6,6 +6,8 @@ import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerType;
 
+import java.util.concurrent.TimeUnit;
+
 @Data
 @Builder
 public class RateLimitsTrigger implements NotificationRuleTrigger {
@@ -22,7 +24,25 @@ public class RateLimitsTrigger implements NotificationRuleTrigger {
 
     @Override
     public EntityId getOriginatorEntityId() {
-        return tenantId;
+        return limitLevel != null ? limitLevel : tenantId;
+    }
+
+
+    private static final long deduplicationDuration = TimeUnit.HOURS.toMillis(1);
+
+    @Override
+    public boolean deduplicate() {
+        return true;
+    }
+
+    @Override
+    public String getDeduplicationKey() {
+        return String.join(":", NotificationRuleTrigger.super.getDeduplicationKey(), api);
+    }
+
+    @Override
+    public long getDeduplicationDuration() {
+        return deduplicationDuration;
     }
 
 }

dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java

@@ -98,6 +98,9 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS
             defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForSysadmin, sysAdmins.getId());
             defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForTenant, affectedTenantAdmins.getId());
 
+            defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId());
+            defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId());
+
             defaultNotifications.create(tenantId, DefaultNotifications.newPlatformVersion, sysAdmins.getId());
             return;
         }

dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java

@@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.EntityAction
 import org.thingsboard.server.common.data.notification.rule.trigger.NewPlatformVersionNotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTriggerType;
+import org.thingsboard.server.common.data.notification.rule.trigger.RateLimitsNotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.rule.trigger.RuleEngineComponentLifecycleEventNotificationRuleTriggerConfig;
 import org.thingsboard.server.common.data.notification.template.NotificationTemplate;
 import org.thingsboard.server.common.data.notification.template.NotificationTemplateConfig;
@@ -134,6 +135,31 @@ public class DefaultNotifications {
                     .description("Send notification to tenant admins when API feature is disabled")
                     .build())
             .build();
+    public static final DefaultNotification exceededRateLimits = DefaultNotification.builder()
+            .name("Exceeded rate limits notification for tenant")
+            .type(NotificationType.RATE_LIMITS)
+            .subject("Rate limits exceeded")
+            .text("Rate limits for ${api} #{ !limitLevelEntityType.isEmpty() ? " +
+                    "'per ${limitLevelEntityType:lowerCase} ' : '' }exceeded" +
+                    "#{ !limitLevelEntityName.isEmpty() ? ' for ${limitLevelEntityType:lowerCase} \\'${limitLevelEntityName}\\'' : " +
+                    " !limitLevelEntityId.isEmpty() ? ' for ${limitLevelEntityType:lowerCase} ${limitLevelEntityId}' : '' : '' }")
+            .icon("block").color("#e91a1a")
+            .rule(DefaultRule.builder()
+                    .name("Rate limits exceeded")
+                    .triggerConfig(new RateLimitsNotificationRuleTriggerConfig())
+                    .description("Send notification to tenant admins when some rate limits are exceeded")
+                    .build())
+            .build();
+    public static final DefaultNotification exceededRateLimitsForSysadmin = exceededRateLimits.toBuilder()
+            .name("Exceeded rate limits notification for sysadmin")
+            .subject("Rate limits exceeded for tenant ${tenantName}")
+            .button("Go to tenant").link("/tenants/${tenantId}")
+            .rule(exceededRateLimits.getRule().toBuilder()
+                    .name("Rate limits exceeded (sysadmin)")
+                    .description("Send notification to system admins when a tenant exceeds some rate limits")
+                    .build())
+            .build();
+
     public static final DefaultNotification newPlatformVersion = DefaultNotification.builder()
             .name("New platform version notification")
             .type(NotificationType.NEW_PLATFORM_VERSION)

dao/src/main/java/org/thingsboard/server/dao/util/limits/DefaultRateLimitService.java

@@ -20,6 +20,7 @@ import com.github.benmanes.caffeine.cache.Caffeine;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.TenantProfile;
@@ -42,7 +43,7 @@ public class DefaultRateLimitService implements RateLimitService {
     private final NotificationRuleProcessor notificationRuleProcessor;
 
     public DefaultRateLimitService(TbTenantProfileCache tenantProfileCache,
-                                   NotificationRuleProcessor notificationRuleProcessor,
+                                   @Lazy NotificationRuleProcessor notificationRuleProcessor,
                                    @Value("${cache.rateLimits.timeToLiveInMinutes:120}") int rateLimitsTtl,
                                    @Value("${cache.rateLimits.maxSize:200000}") int rateLimitsCacheMaxSize) {
         this.tenantProfileCache = tenantProfileCache;