Merge pull request #15033 from thingsboard/master-rc

RC

application/src/main/resources/thingsboard.yml

@@ -197,6 +197,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Statistics reporting interval, set to send summarized data every 10 seconds by default
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
     check:

application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@@ -430,6 +430,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
         tenantProfileService.deleteTenantProfiles(TenantId.SYS_TENANT_ID);
 
         jdbcTemplate.execute("TRUNCATE TABLE notification");
+        jdbcTemplate.execute("TRUNCATE TABLE audit_log");
 
         log.debug("Executed web test teardown");
     }

application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java

@@ -407,20 +407,15 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         testBroadcastEntityStateChangeEventNeverTenantProfile();
     }
 
-    private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) throws Exception {
+    private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) {
         Awaitility.await(awaitMessage)
                 .atMost(TIMEOUT, TimeUnit.SECONDS)
-                .until(() ->
-                        doGetTypedWithTimePageLink(
-                                "/api/audit/logs/entity/TENANT_PROFILE/" + tenantProfileId.getId() + "?",
-                                new TypeReference<PageData<AuditLog>>() {
-                                },
-                                new TimePageLink(5))
-                                .getData()
-                                .stream()
-                                .anyMatch(log -> log.getActionType() == expectedAction)
+                .until(() -> doGetTypedWithTimePageLink(
+                        "/api/audit/logs?",
+                        new TypeReference<PageData<AuditLog>>() {},
+                        new TimePageLink(100)).getData().stream()
+                        .anyMatch(log -> log.getEntityId().equals(tenantProfileId) && log.getActionType() == expectedAction)
                 );
-
     }
 
     private TenantProfile createTenantProfile(String name) {

application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java

@@ -19,6 +19,8 @@ import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.TestPropertySource;
+import org.thingsboard.server.common.data.ApiUsageRecordKey;
+import org.thingsboard.server.common.data.ApiUsageState;
 import org.thingsboard.server.common.data.ApiUsageStateValue;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest;
@@ -30,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
 import org.thingsboard.server.common.data.security.DeviceCredentialsType;
 import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
 import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
+import org.thingsboard.server.common.stats.TbApiUsageReportClient;
 import org.thingsboard.server.controller.AbstractControllerTest;
 import org.thingsboard.server.controller.TbUrlConstants;
 import org.thingsboard.server.dao.service.DaoSqlTest;
@@ -46,6 +49,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 @TestPropertySource(properties = {
         "usage.stats.report.enabled=true",
         "usage.stats.report.interval=2",
+        "usage.stats.report.urgent_interval=1",
         "usage.stats.gauge_report_interval=1",
 })
 public class ApiUsageTest extends AbstractControllerTest {
@@ -54,9 +58,12 @@ public class ApiUsageTest extends AbstractControllerTest {
     private User tenantAdmin;
 
     private static final int MAX_DP_ENABLE_VALUE = 12;
+    private static final int MAX_SMS_ENABLE_VALUE = 10;
     private static final double WARN_THRESHOLD_VALUE = 0.5;
     @Autowired
     private ApiUsageStateService apiUsageStateService;
+    @Autowired
+    private TbApiUsageReportClient apiUsageReportClient;
 
     @Before
     public void beforeTest() throws Exception {
@@ -82,7 +89,7 @@ public class ApiUsageTest extends AbstractControllerTest {
     }
 
     @Test
-    public void testTelemetryApiCall() throws Exception {
+    public void testDbStorageApiUsage() throws Exception {
         Device device = createDevice();
         assertNotNull(device);
         String telemetryPayload = "{\"temperature\":25, \"humidity\":60}";
@@ -94,7 +101,8 @@ public class ApiUsageTest extends AbstractControllerTest {
             doPostAsync(url, telemetryPayload, String.class, status().isOk());
         }
 
-        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(ApiUsageStateValue.WARNING, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()));
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
+                assertEquals(ApiUsageStateValue.WARNING, getUsageState().getDbStorageState()));
 
         long VALUE_DISABLE = (long) (MAX_DP_ENABLE_VALUE - (MAX_DP_ENABLE_VALUE * WARN_THRESHOLD_VALUE)) / 2;
 
@@ -104,10 +112,35 @@ public class ApiUsageTest extends AbstractControllerTest {
 
         await().atMost(TIMEOUT, TimeUnit.SECONDS)
                 .untilAsserted(() -> {
-                    assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState());
+                    assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getDbStorageState());
                 });
     }
 
+    @Test
+    public void testSmsApiUsage() {
+        long smsWarnThreshold = (long) (MAX_SMS_ENABLE_VALUE * WARN_THRESHOLD_VALUE);
+
+        for (int i = 0; i < smsWarnThreshold; i++) {
+            apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT);
+        }
+
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
+                assertEquals(ApiUsageStateValue.WARNING, getUsageState().getSmsExecState()));
+
+        long smsDisableCount = MAX_SMS_ENABLE_VALUE - smsWarnThreshold;
+
+        for (int i = 0; i < smsDisableCount; i++) {
+            apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT);
+        }
+
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
+                assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getSmsExecState()));
+    }
+
+    private ApiUsageState getUsageState() {
+        return apiUsageStateService.findTenantApiUsageState(tenantId);
+    }
+
     private TenantProfile createTenantProfile() {
         TenantProfile tenantProfile = new TenantProfile();
         tenantProfile.setName("Tenant Profile");
@@ -116,6 +149,8 @@ public class ApiUsageTest extends AbstractControllerTest {
         TenantProfileData tenantProfileData = new TenantProfileData();
         DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder()
                 .maxDPStorageDays(MAX_DP_ENABLE_VALUE)
+                .maxSms(MAX_SMS_ENABLE_VALUE)
+                .smsEnabled(true)
                 .warnThreshold(WARN_THRESHOLD_VALUE)
                 .build();
 

application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java

@@ -52,6 +52,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 @TestPropertySource(properties = {
         "usage.stats.report.enabled=true",
         "usage.stats.report.interval=1",
+        "usage.stats.report.urgent_interval=1"
 })
 public class DefaultSmsServiceTest extends AbstractControllerTest {
     @SpyBean

application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java

@@ -41,6 +41,7 @@ import static org.awaitility.Awaitility.await;
 @TestPropertySource(properties = {
         "usage.stats.report.enabled=true",
         "usage.stats.report.interval=2",
+        "usage.stats.report.urgent_interval=1",
         "usage.stats.gauge_report_interval=1",
         "usage.stats.devices.report_interval=3",
         "state.defaultStateCheckIntervalInSec=3",

application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java

@@ -159,14 +159,17 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte
     protected List<Device> connectClientWithCorrectAccessTokenWithNDEATHCreatedDevices(int cntDevices, long ts) throws Exception {
         List<Device> devices = new ArrayList<>();
         clientWithCorrectNodeAccessTokenWithNDEATH();
-        MetricDataType metricDataType = Int32;
-        String key = "Node Metric int32";
+        String keyInt = "Node Metric int32";
         int valueDeviceInt32 = 1024;
-        SparkplugBProto.Payload.Metric metric = createMetric(valueDeviceInt32, ts, key, metricDataType, -1L);
+        SparkplugBProto.Payload.Metric metricInt = createMetric(valueDeviceInt32, ts, keyInt,  Int32, -1L);
+        String keyStringEmpty = "Node Metric String Empty";
+        String valueDeviceStringEmpty = "";
+        SparkplugBProto.Payload.Metric metricStringEmpty = createMetric(valueDeviceStringEmpty, ts, keyStringEmpty, MetricDataType.String, -1L);
         SparkplugBProto.Payload.Builder payloadBirthNode = SparkplugBProto.Payload.newBuilder()
                 .setTimestamp(ts)
                 .setSeq(getBdSeqNum());
-        payloadBirthNode.addMetrics(metric);
+        payloadBirthNode.addMetrics(metricInt);
+        payloadBirthNode.addMetrics(metricStringEmpty);
         payloadBirthNode.setTimestamp(ts);
         if (client.isConnected()) {
             client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.NBIRTH.name() + "/" + edgeNode,
@@ -174,14 +177,14 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte
         }
 
         valueDeviceInt32 = 4024;
-        metric = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L);
+        metricInt = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L);
         for (int i = 0; i < cntDevices; i++) {
             SparkplugBProto.Payload.Builder payloadBirthDevice = SparkplugBProto.Payload.newBuilder()
                     .setTimestamp(ts)
                     .setSeq(getSeqNum());
             String deviceName = deviceId + "_" + i;
 
-            payloadBirthDevice.addMetrics(metric);
+            payloadBirthDevice.addMetrics(metricInt);
             if (client.isConnected()) {
                 client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.DBIRTH.name() + "/" + edgeNode + "/" + deviceName,
                         payloadBirthDevice.build().toByteArray(), 0, false);

common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java

@@ -25,8 +25,8 @@ public enum ApiUsageRecordKey {
     RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit", "Rule Engine execution"),
     JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit", "JavaScript execution"),
     TBEL_EXEC_COUNT(ApiFeature.TBEL, "tbelExecutionCount", "tbelExecutionLimit", "Tbel execution"),
-    EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message"),
-    SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message"),
+    EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message", true, true),
+    SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message", true, true),
     CREATED_ALARMS_COUNT(ApiFeature.ALARM, "createdAlarmsCount", "createdAlarmsLimit", "alarm"),
     ACTIVE_DEVICES("activeDevicesCount"),
     INACTIVE_DEVICES("inactiveDevicesCount");
@@ -50,21 +50,24 @@ public enum ApiUsageRecordKey {
     private final String unitLabel;
     @Getter
     private final boolean counter;
+    @Getter
+    private final boolean urgent; // urgent keys are reported at a shorter interval for quicker usage state updates
 
     ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel) {
-        this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true);
+        this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true, false);
     }
 
     ApiUsageRecordKey(String apiCountKey) {
-        this(null, apiCountKey, null, null, false);
+        this(null, apiCountKey, null, null, false, false);
     }
 
-    ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter) {
+    ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter, boolean urgent) {
         this.apiFeature = apiFeature;
         this.apiCountKey = apiCountKey;
         this.apiLimitKey = apiLimitKey;
         this.unitLabel = unitLabel;
         this.counter = counter;
+        this.urgent = urgent;
     }
 
     public static ApiUsageRecordKey[] getKeys(ApiFeature feature) {

common/edge-api/src/main/proto/edge.proto

@@ -46,12 +46,12 @@ enum EdgeVersion {
   V_4_2_0 = 12;
   V_4_3_0 = 13;
   V_4_2_1_2 = 14;
-  V_4_2_1_3 = 420;
+  V_4_2_1_3 = 4213;
   V_4_3_0_1 = 15;
-  V_4_3_0_2 = 430;
-  V_4_4_0 = 440;
+  V_4_3_0_2 = 4302;
+  V_4_4_0 = 4400;
 
-  V_LATEST = 999;
+  V_LATEST = 99999;
 }
 
 /**

common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java

@@ -156,11 +156,7 @@ public class ProtoConverter {
                 case BOOLEAN_V:
                 case LONG_V:
                 case DOUBLE_V:
-                    break;
                 case STRING_V:
-                    if (StringUtils.isEmpty(keyValueProto.getStringV())) {
-                        throw new IllegalArgumentException("Value is empty for key: " + key + "!");
-                    }
                     break;
                 case JSON_V:
                     try {

common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java

@@ -63,8 +63,10 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient {
     private boolean enabled;
     @Value("${usage.stats.report.enabled_per_customer:false}")
     private boolean enabledPerCustomer;
-    @Value("${usage.stats.report.interval:10}")
+    @Value("${usage.stats.report.interval:60}")
     private int interval;
+    @Value("${usage.stats.report.urgent_interval:10}")
+    private int urgentInterval;
     @Value("${usage.stats.report.pack_size:1024}")
     private int packSize;
 
@@ -83,20 +85,30 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient {
             for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
                 stats.put(key, new ConcurrentHashMap<>());
             }
+            Random random = new Random();
             scheduler.scheduleWithFixedDelay(() -> {
                 try {
-                    reportStats();
+                    reportStats(false);
                 } catch (Exception e) {
                     log.warn("Failed to report statistics: ", e);
                 }
-            }, new Random().nextInt(interval), interval, TimeUnit.SECONDS);
+            }, random.nextInt(interval), interval, TimeUnit.SECONDS);
+            scheduler.scheduleWithFixedDelay(() -> {
+                try {
+                    reportStats(true);
+                } catch (Exception e) {
+                    log.warn("Failed to report urgent statistics: ", e);
+                }
+            }, random.nextInt(urgentInterval), urgentInterval, TimeUnit.SECONDS);
         }
     }
 
-    private void reportStats() {
+    private void reportStats(boolean urgent) {
         ConcurrentMap<ParentEntity, UsageStatsServiceMsg.Builder> report = new ConcurrentHashMap<>();
 
         for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
+            if (key.isUrgent() != urgent) continue;
+
             ConcurrentMap<ReportLevel, AtomicLong> statsForKey = stats.get(key);
             statsForKey.forEach((reportLevel, statsValue) -> {
                 long value = statsValue.get();

msa/vc-executor/src/main/resources/tb-vc-executor.yml

@@ -215,6 +215,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
 

transport/coap/src/main/resources/tb-coap-transport.yml

@@ -421,6 +421,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
 

transport/http/src/main/resources/tb-http-transport.yml

@@ -370,6 +370,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
 

transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml

@@ -471,6 +471,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
 

transport/mqtt/src/main/resources/tb-mqtt-transport.yml

@@ -404,6 +404,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
 

transport/snmp/src/main/resources/tb-snmp-transport.yml

@@ -359,6 +359,8 @@ usage:
       enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
       # Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
       interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
+      # Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
+      urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
       # Amount of statistic messages in pack
       pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"