Merge with rc

application/src/main/data/json/system/widget_types/gateway_status.json

@@ -0,0 +1,44 @@
+{
+  "fqn": "gateway_widgets.gateway_status",
+  "name": "Gateway status",
+  "deprecated": false,
+  "image": "tb-image;/api/images/system/gateway_status_system_widget_image.png",
+  "description": "Indicator of Gateway connection status.",
+  "descriptor": {
+    "type": "latest",
+    "sizeX": 7.5,
+    "sizeY": 3,
+    "resources": [
+      {
+        "url": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js",
+        "isModule": true
+      }
+    ],
+    "templateHtml": "<tb-gateway-status [deviceId]=\"entityId\" *ngIf=\"entityId\" [ctx]=\"ctx\"></tb-gateway-status>",
+    "templateCss": "",
+    "controllerScript": "self.onInit = function() {\n    if (self.ctx.datasources && self.ctx.datasources.length) {\n        self.ctx.$scope.entityId = self.ctx.datasources[0].entity.id;\n    }\n};\n\nself.onDataUpdated = function() {\n    self.ctx.$scope.gatewayStatus?.onDataUpdated();\n};",
+    "settingsSchema": "",
+    "dataKeySettingsSchema": "",
+    "settingsDirective": "tb-gateway-status",
+    "defaultConfig": "{\"datasources\":[{\"type\":\"static\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Random\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = prevValue + Math.random() * 100 - 50;\\nvar multiplier = Math.pow(10, 2 || 0);\\nvar value = Math.round(value * multiplier) / multiplier;\\nif (value < -1000) {\\n\\tvalue = -1000;\\n} else if (value > 1000) {\\n\\tvalue = 1000;\\n}\\nreturn value;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":false,\"backgroundColor\":\"rgb(255, 255, 255)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{\"cardHtml\":\"<div class='card'>HTML code here</div>\",\"cardCss\":\".card {\\n    font-weight: bold;\\n    font-size: 32px;\\n    color: #999;\\n    width: 100%;\\n    height: 100%;\\n    display: flex;\\n    align-items: center;\\n    justify-content: center;\\n}\"},\"title\":\"HTML Card\",\"dropShadow\":true}"
+  },
+  "tags": [
+    "router",
+    "bridge",
+    "hub",
+    "access point",
+    "relay",
+    "opc ua",
+    "opc-ua",
+    "modbus",
+    "bacnet",
+    "odbc",
+    "ftp",
+    "snmp",
+    "mqtt",
+    "xmpp",
+    "ocpp",
+    "ble",
+    "bluetooth"
+  ]
+}

application/src/main/data/upgrade/basic/schema_update.sql

@@ -23,7 +23,7 @@ UPDATE user_credentials c SET failed_login_attempts = (SELECT (additional_info::
   WHERE failed_login_attempts IS NULL;
 
 UPDATE tb_user SET additional_info = (additional_info::jsonb - 'lastLoginTs' - 'failedLoginAttempts' - 'userCredentialsEnabled')::text
-  WHERE additional_info IS NOT NULL AND additional_info != 'null';
+  WHERE additional_info IS NOT NULL AND additional_info != 'null' AND jsonb_typeof(additional_info::jsonb) = 'object';
 
 -- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY START
 

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -35,6 +35,8 @@ import org.thingsboard.server.service.install.migrate.TsLatestMigrateService;
 import org.thingsboard.server.service.install.update.CacheCleanupService;
 import org.thingsboard.server.service.install.update.DataUpdateService;
 
+import static org.thingsboard.server.service.install.update.DefaultDataUpdateService.getEnv;
+
 @Service
 @Profile("install")
 @Slf4j
@@ -94,11 +96,11 @@ public class ThingsboardInstallService {
     public void performInstall() {
         try {
             if (isUpgrade) {
-                log.info("Starting ThingsBoard Upgrade from version {} ...", upgradeFromVersion);
-
                 if ("cassandra-latest-to-postgres".equals(upgradeFromVersion)) {
                     log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ...");
                     latestMigrateService.migrate();
+                } else if (upgradeFromVersion.equals("3.9.0-resources")) {
+                    installScripts.updateResourcesUsage();
                 } else {
                     // TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM in DefaultDatabaseSchemaSettingsService
                     databaseSchemaVersionService.validateSchemaSettings();
@@ -118,6 +120,14 @@ public class ThingsboardInstallService {
                     entityDatabaseSchemaService.createDatabaseIndexes();
                     // Runs upgrade scripts that are not possible in plain SQL.
                     // TODO: cleanup update code after each release
+                    if (!getEnv("SKIP_RESOURCES_USAGE_MIGRATION", false)) {
+                        installScripts.setUpdateResourcesUsage(true);
+                    } else {
+                        log.info("Skipping resources usage migration. Run the upgrade with fromVersion as '3.9.0-resources' to migrate");
+                    }
+                    if (installScripts.isUpdateResourcesUsage()) {
+                        installScripts.updateResourcesUsage();
+                    }
                     dataUpdateService.updateData();
                     log.info("Updating system data...");
                     dataUpdateService.upgradeRuleNodes();

application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java

@@ -349,7 +349,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
     private void startSyncProcess(TenantId tenantId, EdgeId edgeId, UUID requestId, String requestServiceId) {
         EdgeGrpcSession session = sessions.get(edgeId);
         if (session != null) {
-            if (!session.isSyncCompleted()) {
+            if (session.isSyncInProgress()) {
                 clusterService.pushEdgeSyncResponseToCore(new FromEdgeSyncResponse(requestId, tenantId, edgeId, false, "Sync process is active at the moment"), requestServiceId);
             } else {
                 boolean success = false;
@@ -368,7 +368,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
 
         UUID requestId = request.getId();
         EdgeGrpcSession session = sessions.get(request.getEdgeId());
-        if (session != null && !session.isSyncCompleted()) {
+        if (session != null && session.isSyncInProgress()) {
             responseConsumer.accept(new FromEdgeSyncResponse(requestId, request.getTenantId(), request.getEdgeId(), false, "Sync process is active at the moment"));
         } else {
             log.trace("[{}][{}] Processing sync edge request [{}], serviceId [{}]", request.getTenantId(), request.getId(), request.getEdgeId(), request.getServiceId());
@@ -411,7 +411,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
                     newEventLock.lock();
                     try {
                         if (Boolean.TRUE.equals(sessionNewEvents.get(edgeId))) {
-                            log.trace("[{}][{}] Set session new events flag to false", tenantId, edgeId.getId());
+                            log.trace("[{}][{}] set session new events flag to false", tenantId, edgeId.getId());
                             sessionNewEvents.put(edgeId, false);
                             session.processHighPriorityEvents();
                             processEdgeEventMigrationIfNeeded(session, edgeId);
@@ -420,6 +420,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
                                     @Override
                                     public void onSuccess(Boolean newEventsAdded) {
                                         if (Boolean.TRUE.equals(newEventsAdded)) {
+                                            log.trace("[{}][{}] new events added. set session new events flag to true", tenantId, edgeId.getId());
                                             sessionNewEvents.put(edgeId, true);
                                         }
                                         scheduleEdgeEventsCheck(session);

application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@@ -28,6 +28,7 @@ import org.springframework.data.util.Pair;
 import org.thingsboard.server.common.data.AttributeScope;
 import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.EdgeUtils;
+import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.edge.Edge;
 import org.thingsboard.server.common.data.edge.EdgeEvent;
 import org.thingsboard.server.common.data.id.EdgeId;
@@ -130,13 +131,12 @@ public abstract class EdgeGrpcSession implements Closeable {
     private Long previousStartTs;
     private Long newStartSeqId;
     private Long previousStartSeqId;
-    private Long seqIdEnd;
 
     private StreamObserver<RequestMsg> inputStream;
     private StreamObserver<ResponseMsg> outputStream;
 
     private volatile boolean connected;
-    private volatile boolean syncCompleted;
+    private volatile boolean syncInProgress;
 
     private EdgeVersion edgeVersion;
     private int maxInboundMessageSize;
@@ -191,7 +191,7 @@ public abstract class EdgeGrpcSession implements Closeable {
                             }
                             startSyncProcess(fullSync);
                         } else {
-                            syncCompleted = true;
+                            syncInProgress = false;
                         }
                     }
                     if (requestMsg.getMsgType().equals(RequestMsgType.UPLINK_RPC_MESSAGE)) {
@@ -246,10 +246,14 @@ public abstract class EdgeGrpcSession implements Closeable {
     }
 
     public void startSyncProcess(boolean fullSync) {
-        log.info("[{}][{}][{}] Staring edge sync process", tenantId, edge.getId(), sessionId);
-        syncCompleted = false;
-        interruptGeneralProcessingOnSync();
-        doSync(new EdgeSyncCursor(ctx, edge, fullSync));
+        if (!syncInProgress) {
+            log.info("[{}][{}][{}] Staring edge sync process", tenantId, edge.getId(), sessionId);
+            syncInProgress = true;
+            interruptGeneralProcessingOnSync();
+            doSync(new EdgeSyncCursor(ctx, edge, fullSync));
+        } else {
+            log.info("[{}][{}][{}] Sync is already started, skipping starting it now", tenantId, edge.getId(), sessionId);
+        }
     }
 
     private void doSync(EdgeSyncCursor cursor) {
@@ -292,6 +296,7 @@ public abstract class EdgeGrpcSession implements Closeable {
 
     protected void processEdgeEvents(EdgeEventFetcher fetcher, PageLink pageLink, SettableFuture<Pair<Long, Long>> result) {
         try {
+            log.trace("[{}] Start processing edge events, fetcher = {}, pageLink = {}", sessionId, fetcher.getClass().getSimpleName(), pageLink);
             processHighPriorityEvents();
             PageData<EdgeEvent> pageData = fetcher.fetchEdgeEvents(edge.getTenantId(), edge, pageLink);
             if (isConnected() && !pageData.getData().isEmpty()) {
@@ -327,7 +332,7 @@ public abstract class EdgeGrpcSession implements Closeable {
                     }
                 }, ctx.getGrpcCallbackExecutorService());
             } else {
-                log.trace("[{}] no event(s) found. Stop processing edge events", sessionId);
+                log.trace("[{}] no event(s) found. Stop processing edge events, fetcher = {}, pageLink = {}", sessionId, fetcher.getClass().getSimpleName(), pageLink);
                 result.set(null);
             }
         } catch (Exception e) {
@@ -406,6 +411,8 @@ public abstract class EdgeGrpcSession implements Closeable {
                 || sessionState.getScheduledSendDownlinkTask() != null && !sessionState.getScheduledSendDownlinkTask().isCancelled()) {
             log.debug("[{}][{}][{}] Previous send downlink future was not properly completed, stopping it now!", tenantId, edge.getId(), sessionId);
             stopCurrentSendDownlinkMsgsTask(true);
+        } else {
+            log.trace("[{}][{}][{}] Previous send downlink future is not active", tenantId, edge.getId(), sessionId);
         }
     }
 
@@ -519,12 +526,12 @@ public abstract class EdgeGrpcSession implements Closeable {
         try {
             if (msg.getSuccess()) {
                 sessionState.getPendingMsgsMap().remove(msg.getDownlinkMsgId());
-                log.debug("[{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg);
+                log.debug("[{}][{}][{}] Msg has been processed successfully! Msg Id: [{}], Msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg);
             } else {
-                log.error("[{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getRoutingKey(), msg.getDownlinkMsgId(), msg.getErrorMsg());
+                log.error("[{}][{}][{}] Msg processing failed! Msg Id: [{}], Error msg: {}", tenantId, edge.getId(), sessionId, msg.getDownlinkMsgId(), msg.getErrorMsg());
             }
             if (sessionState.getPendingMsgsMap().isEmpty()) {
-                log.debug("[{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getRoutingKey());
+                log.debug("[{}][{}][{}] Pending msgs map is empty. Stopping current iteration", tenantId, edge.getId(), sessionId);
                 stopCurrentSendDownlinkMsgsTask(false);
             }
         } catch (Exception e) {
@@ -534,7 +541,7 @@ public abstract class EdgeGrpcSession implements Closeable {
 
     public void processHighPriorityEvents() {
         try {
-            if (isConnected() && isSyncCompleted()) {
+            if (isConnected() && !isSyncInProgress()) {
                 if (highPriorityQueue.isEmpty()) {
                     return;
                 }
@@ -543,6 +550,7 @@ public abstract class EdgeGrpcSession implements Closeable {
                 while ((event = highPriorityQueue.poll()) != null) {
                     highPriorityEvents.add(event);
                 }
+                log.trace("[{}][{}] Sending high priority events {}", tenantId, sessionId, highPriorityEvents.size());
                 List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(highPriorityEvents);
                 sendDownlinkMsgsPack(downlinkMsgsPack).get();
             }
@@ -553,18 +561,18 @@ public abstract class EdgeGrpcSession implements Closeable {
 
     public ListenableFuture<Boolean> processEdgeEvents() throws Exception {
         SettableFuture<Boolean> result = SettableFuture.create();
-        log.trace("[{}][{}] starting processing edge events", tenantId, sessionId);
-        if (isConnected() && isSyncCompleted()) {
+        if (isConnected() && !isSyncInProgress()) {
             Pair<Long, Long> startTsAndSeqId = getQueueStartTsAndSeqId().get();
             previousStartTs = startTsAndSeqId.getFirst();
             previousStartSeqId = startTsAndSeqId.getSecond();
             GeneralEdgeEventFetcher fetcher = new GeneralEdgeEventFetcher(
                     previousStartTs,
                     previousStartSeqId,
-                    seqIdEnd,
                     false,
                     Integer.toUnsignedLong(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount()),
                     ctx.getEdgeEventService());
+            log.trace("[{}][{}] starting processing edge events, previousStartTs = {}, previousStartSeqId = {}",
+                    tenantId, sessionId, previousStartTs, previousStartSeqId);
             Futures.addCallback(startProcessingEdgeEvents(fetcher), new FutureCallback<>() {
                 @Override
                 public void onSuccess(@Nullable Pair<Long, Long> newStartTsAndSeqId) {
@@ -574,18 +582,16 @@ public abstract class EdgeGrpcSession implements Closeable {
                             @Override
                             public void onSuccess(@Nullable List<Long> list) {
                                 log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId);
+                                boolean newEventsAvailable;
                                 if (fetcher.isSeqIdNewCycleStarted()) {
-                                    seqIdEnd = fetcher.getSeqIdEnd();
-                                    boolean newEventsAvailable = isNewEdgeEventsAvailable();
-                                    result.set(newEventsAvailable);
+                                    newEventsAvailable = isNewEdgeEventsAvailable();
                                 } else {
-                                    seqIdEnd = null;
-                                    boolean newEventsAvailable = isSeqIdStartedNewCycle();
+                                    newEventsAvailable = isSeqIdStartedNewCycle();
                                     if (!newEventsAvailable) {
                                         newEventsAvailable = isNewEdgeEventsAvailable();
                                     }
-                                    result.set(newEventsAvailable);
                                 }
+                                result.set(newEventsAvailable);
                             }
 
                             @Override
@@ -607,7 +613,7 @@ public abstract class EdgeGrpcSession implements Closeable {
                 }
             }, ctx.getGrpcCallbackExecutorService());
         } else {
-            if (!isSyncCompleted()) {
+            if (isSyncInProgress()) {
                 log.trace("[{}][{}] edge sync is not completed yet. Skipping iteration", tenantId, sessionId);
                 result.set(Boolean.TRUE);
             } else {
@@ -672,9 +678,12 @@ public abstract class EdgeGrpcSession implements Closeable {
 
     private boolean isSeqIdStartedNewCycle() {
         try {
+            log.trace("[{}][{}][{}] Checking if seq id started new cycle", tenantId, edge.getId(), sessionId);
             TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis());
             PageData<EdgeEvent> edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), 0L, previousStartSeqId == 0 ? null : previousStartSeqId - 1, pageLink);
-            return !edgeEvents.getData().isEmpty();
+            boolean result = !edgeEvents.getData().isEmpty();
+            log.trace("[{}][{}][{}] Result of check if seq id started new cycle, result = {}", tenantId, edge.getId(), sessionId, result);
+            return result;
         } catch (Exception e) {
             log.error("[{}][{}][{}] Failed to execute isSeqIdStartedNewCycle", tenantId, edge.getId(), sessionId, e);
         }
@@ -683,9 +692,12 @@ public abstract class EdgeGrpcSession implements Closeable {
 
     private boolean isNewEdgeEventsAvailable() {
         try {
+            log.trace("[{}][{}][{}] Checking if new edge events available", tenantId, edge.getId(), sessionId);
             TimePageLink pageLink = new TimePageLink(ctx.getEdgeEventStorageSettings().getMaxReadRecordsCount(), 0, null, null, newStartTs, System.currentTimeMillis());
             PageData<EdgeEvent> edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), newStartSeqId, null, pageLink);
-            return !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty();
+            boolean result = !edgeEvents.getData().isEmpty() || !highPriorityQueue.isEmpty();
+            log.trace("[{}][{}][{}] Result of check if new edge events available, result = {}", tenantId, edge.getId(), sessionId, result);
+            return result;
         } catch (Exception e) {
             log.error("[{}][{}][{}] Failed to execute isNewEdgeEventsAvailable", tenantId, edge.getId(), sessionId, e);
         }
@@ -724,7 +736,7 @@ public abstract class EdgeGrpcSession implements Closeable {
     }
 
     private void markSyncCompletedSendEdgeEventUpdate() {
-        syncCompleted = true;
+        syncInProgress = false;
         ctx.getClusterService().onEdgeEventUpdate(new EdgeEventUpdateMsg(edge.getTenantId(), edge.getId()));
     }
 
@@ -737,29 +749,27 @@ public abstract class EdgeGrpcSession implements Closeable {
         }
     }
 
-    private void sendDownlinkMsg(ResponseMsg downlinkMsg) {
-        if (downlinkMsg.getDownlinkMsg().getWidgetTypeUpdateMsgCount() > 0) {
-            log.trace("[{}][{}] Sending downlink widgetTypeUpdateMsg, downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId());
-        } else {
-            log.trace("[{}][{}] Sending downlink msg [{}]", tenantId, sessionId, downlinkMsg);
-        }
+    private void sendDownlinkMsg(ResponseMsg responseMsg) {
         if (isConnected()) {
+            String responseMsgStr = StringUtils.truncate(responseMsg.toString(), 10000);
+            log.trace("[{}][{}] Sending downlink msg [{}]", tenantId, sessionId, responseMsgStr);
             downlinkMsgLock.lock();
+            String downlinkMsgStr = responseMsg.hasDownlinkMsg() ? String.valueOf(responseMsg.getDownlinkMsg().getDownlinkMsgId()) : responseMsgStr;
             try {
-                outputStream.onNext(downlinkMsg);
+                outputStream.onNext(responseMsg);
             } catch (Exception e) {
-                log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsg, e);
+                log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsgStr, e);
                 connected = false;
                 sessionCloseListener.accept(edge, sessionId);
             } finally {
                 downlinkMsgLock.unlock();
             }
-            log.trace("[{}][{}] Response msg successfully sent. downlinkMsgId = {}", tenantId, sessionId, downlinkMsg.getDownlinkMsg().getDownlinkMsgId());
+            log.trace("[{}][{}] downlink msg successfully sent [{}]", tenantId, sessionId, downlinkMsgStr);
         }
     }
 
-    protected DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) {
-        log.trace("[{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", edgeEvent.getTenantId(), edgeEvent, edgeEvent.getAction());
+    private DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) {
+        log.trace("[{}][{}] Executing convertEntityEventToDownlink, edgeEvent [{}], action [{}]", tenantId, sessionId, edgeEvent, edgeEvent.getAction());
         return switch (edgeEvent.getType()) {
             case EDGE -> ctx.getEdgeProcessor().convertEdgeEventToDownlink(edgeEvent);
             case DEVICE -> ctx.getDeviceProcessor().convertDeviceEventToDownlink(edgeEvent, edgeVersion);
@@ -789,7 +799,7 @@ public abstract class EdgeGrpcSession implements Closeable {
             case OAUTH2_CLIENT -> ctx.getOAuth2EdgeProcessor().convertOAuth2ClientEventToDownlink(edgeEvent, edgeVersion);
             case DOMAIN -> ctx.getOAuth2EdgeProcessor().convertOAuth2DomainEventToDownlink(edgeEvent, edgeVersion);
             default -> {
-                log.warn("[{}] Unsupported edge event type [{}]", edgeEvent.getTenantId(), edgeEvent);
+                log.warn("[{}][{}] Unsupported edge event type [{}]", tenantId, sessionId, edgeEvent);
                 yield null;
             }
         };
@@ -923,8 +933,8 @@ public abstract class EdgeGrpcSession implements Closeable {
             }
         } catch (Exception e) {
             String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg);
-            log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e);
-            ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId())
+            log.error("[{}][{}] Can't process uplink msg [{}]", tenantId, sessionId, uplinkMsg, e);
+            ctx.getNotificationRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(tenantId).edgeId(edge.getId())
                     .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build());
             return Futures.immediateFailedFuture(e);
         }

application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java

@@ -73,7 +73,7 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession {
 
     private void processMsgs(List<TbProtoQueueMsg<ToEdgeEventNotificationMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeEventNotificationMsg>> consumer) {
         log.trace("[{}][{}] starting processing edge events", tenantId, sessionId);
-        if (isConnected() && isSyncCompleted() && !isHighPriorityProcessing) {
+        if (isConnected() && !isSyncInProgress() && !isHighPriorityProcessing) {
             List<EdgeEvent> edgeEvents = new ArrayList<>();
             for (TbProtoQueueMsg<ToEdgeEventNotificationMsg> msg : msgs) {
                 EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg());

application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java

@@ -33,8 +33,6 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher {
     private final Long queueStartTs;
     private Long seqIdStart;
     @Getter
-    private Long seqIdEnd;
-    @Getter
     private boolean seqIdNewCycleStarted;
     private Long maxReadRecordsCount;
     private final EdgeEventService edgeEventService;
@@ -53,10 +51,11 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher {
     @Override
     public PageData<EdgeEvent> fetchEdgeEvents(TenantId tenantId, Edge edge, PageLink pageLink) {
         try {
-            PageData<EdgeEvent> edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), seqIdStart, seqIdEnd, (TimePageLink) pageLink);
+            log.trace("[{}] Finding general edge events [{}], seqIdStart = {}, pageLink = {}",
+                    tenantId, edge.getId(), seqIdStart, pageLink);
+            PageData<EdgeEvent> edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), seqIdStart, null, (TimePageLink) pageLink);
             if (edgeEvents.getData().isEmpty()) {
-                this.seqIdEnd = Math.max(this.maxReadRecordsCount, seqIdStart - this.maxReadRecordsCount);
-                edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), 0L, seqIdEnd, (TimePageLink) pageLink);
+                edgeEvents = edgeEventService.findEdgeEvents(tenantId, edge.getId(), 0L, Math.max(this.maxReadRecordsCount, seqIdStart - this.maxReadRecordsCount), (TimePageLink) pageLink);
                 if (edgeEvents.getData().stream().anyMatch(ee -> ee.getSeqId() < seqIdStart)) {
                     log.info("[{}] seqId column of edge_event table started new cycle [{}]", tenantId, edge.getId());
                     this.seqIdNewCycleStarted = true;

application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/relation/RelationEdgeProcessor.java

@@ -73,7 +73,7 @@ public abstract class RelationEdgeProcessor extends BaseRelationProcessor implem
 
     public ListenableFuture<Void> processRelationNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) {
         EntityRelation relation = JacksonUtil.fromString(edgeNotificationMsg.getBody(), EntityRelation.class);
-        if (relation == null || (relation.getFrom().getEntityType().equals(EntityType.EDGE) || relation.getTo().getEntityType().equals(EntityType.EDGE))) {
+        if (relation == null) {
             return Futures.immediateFuture(null);
         }
         EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB());

application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java

@@ -235,6 +235,9 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
                     if (relationsList != null && !relationsList.isEmpty()) {
                         List<ListenableFuture<Void>> futures = new ArrayList<>();
                         for (List<EntityRelation> entityRelations : relationsList) {
+                            if (entityRelations.isEmpty()) {
+                                continue;
+                            }
                             log.trace("[{}][{}][{}][{}] relation(s) are going to be pushed to edge.", tenantId, edge.getId(), entityId, entityRelations.size());
                             for (EntityRelation relation : entityRelations) {
                                 try {
@@ -255,19 +258,23 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
                                 }
                             }
                         }
-                        Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() {
-                            @Override
-                            public void onSuccess(@Nullable List<Void> voids) {
-                                futureToSet.set(null);
-                            }
+                        if (futures.isEmpty()) {
+                            futureToSet.set(null);
+                        } else {
+                            Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() {
+                                @Override
+                                public void onSuccess(@Nullable List<Void> voids) {
+                                    futureToSet.set(null);
+                                }
 
-                            @Override
-                            public void onFailure(Throwable throwable) {
-                                String errMsg = String.format("[%s][%s] Exception during saving edge events [%s]!", tenantId, edge.getId(), relationRequestMsg);
-                                log.error(errMsg, throwable);
-                                futureToSet.setException(new RuntimeException(errMsg, throwable));
-                            }
-                        }, dbCallbackExecutorService);
+                                @Override
+                                public void onFailure(Throwable throwable) {
+                                    String errMsg = String.format("[%s][%s] Exception during saving edge events [%s]!", tenantId, edge.getId(), relationRequestMsg);
+                                    log.error(errMsg, throwable);
+                                    futureToSet.setException(new RuntimeException(errMsg, throwable));
+                                }
+                            }, dbCallbackExecutorService);
+                        }
                     } else {
                         futureToSet.set(null);
                     }

application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java

@@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.ResourceSubType;
 import org.thingsboard.server.common.data.ResourceType;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.msg.queue.ServiceType;
@@ -74,7 +75,7 @@ public class DashboardSyncService {
         List<RepoFile> resources = listFiles("resources");
         for (RepoFile resourceFile : resources) {
             byte[] data = getFileContent(resourceFile.path());
-            resourceService.createOrUpdateSystemResource(ResourceType.JS_MODULE, resourceFile.name(), data);
+            resourceService.createOrUpdateSystemResource(ResourceType.JS_MODULE, ResourceSubType.EXTENSION, resourceFile.name(), data);
         }
         List<RepoFile> images = listFiles("images");
         for (RepoFile imageFile : images) {
@@ -89,7 +90,7 @@ public class DashboardSyncService {
         widgetsBundleService.updateSystemWidgets(widgetsBundles, widgetTypes);
 
         RepoFile dashboardFile = listFiles("dashboards").get(0);
-        resourceService.createOrUpdateSystemResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, getFileContent(dashboardFile.path()));
+        resourceService.createOrUpdateSystemResource(ResourceType.DASHBOARD, null, GATEWAYS_DASHBOARD_KEY, getFileContent(dashboardFile.path()));
 
         log.info("Gateways dashboard sync completed");
     }

application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java

@@ -17,6 +17,8 @@ package org.thingsboard.server.service.install;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.info.BuildProperties;
 import org.springframework.context.annotation.Profile;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -39,6 +41,9 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti
     private final BuildProperties buildProperties;
     private final JdbcTemplate jdbcTemplate;
 
+    @Value("${install.upgrade.from_version:}")
+    private String upgradeFromVersion;
+
     private String packageSchemaVersion;
     private String schemaVersionFromDb;
 
@@ -102,6 +107,15 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti
     @Override
     public String getDbSchemaVersion() {
         if (schemaVersionFromDb == null) {
+            if (StringUtils.isNotBlank(upgradeFromVersion)) {
+                /*
+                * TODO - Remove after the release of 3.9.0:
+                *  This a temporary workaround due to the issue that schema version in the
+                *  tb_schema_settings was set as 3.6.4 during the install of 3.8.1.
+                * */
+                schemaVersionFromDb = upgradeFromVersion;
+                return schemaVersionFromDb;
+            }
             Long version = getSchemaVersionFromDb();
             if (version == null) {
                 onSchemaSettingsError("Upgrade failed: the database schema version is missing.");

application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@@ -123,6 +123,9 @@ public class InstallScripts {
     @Getter @Setter
     private boolean updateImages = false;
 
+    @Getter @Setter
+    private boolean updateResourcesUsage = false;
+
     @Autowired
     private ImageService imageService;
 
@@ -416,9 +419,9 @@ public class InstallScripts {
         }
 
         Path resourcesDir = Path.of(getDataDir(), RESOURCES_DIR);
-        loadSystemResources(resourcesDir.resolve("images"), ResourceType.IMAGE);
-        loadSystemResources(resourcesDir.resolve("js_modules"), ResourceType.JS_MODULE);
-        loadSystemResources(resourcesDir.resolve("dashboards"), ResourceType.DASHBOARD);
+        loadSystemResources(resourcesDir.resolve("images"), ResourceType.IMAGE, null);
+        loadSystemResources(resourcesDir.resolve("js_modules"), ResourceType.JS_MODULE, ResourceSubType.EXTENSION);
+        loadSystemResources(resourcesDir.resolve("dashboards"), ResourceType.DASHBOARD, null);
     }
 
     public void loadDashboards(TenantId tenantId, CustomerId customerId) {
@@ -514,7 +517,7 @@ public class InstallScripts {
         resourcesUpdater.updateWidgetsResources();
     }
 
-    private void loadSystemResources(Path dir, ResourceType resourceType) {
+    private void loadSystemResources(Path dir, ResourceType resourceType, ResourceSubType resourceSubType) {
         listDir(dir).forEach(resourceFile -> {
             String resourceKey = resourceFile.getFileName().toString();
             try {
@@ -522,7 +525,7 @@ public class InstallScripts {
                 if (resourceType == ResourceType.IMAGE) {
                     imageService.createOrUpdateSystemImage(resourceKey, data);
                 } else {
-                    resourceService.createOrUpdateSystemResource(resourceType, resourceKey, data);
+                    resourceService.createOrUpdateSystemResource(resourceType, resourceSubType, resourceKey, data);
                 }
             } catch (Exception e) {
                 throw new RuntimeException("Unable to load system resource " + resourceFile, e);

application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@@ -65,7 +65,6 @@ public class DefaultDataUpdateService implements DataUpdateService {
     public void updateData() throws Exception {
         log.info("Updating data ...");
         //TODO: should be cleaned after each release
-        installScripts.updateResourcesUsage();
         log.info("Data updated.");
     }
 

application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java

@@ -113,7 +113,7 @@ public class ResourcesUpdater {
         }
 
         executor.shutdown();
-        if (!executor.awaitTermination(10, TimeUnit.MINUTES)) {
+        if (!executor.awaitTermination(5, TimeUnit.HOURS)) {
             throw new RuntimeException("Dashboards resources update timeout"); // just in case, should happen
         }
         log.info("Updated {} dashboards", updatedCount);
@@ -142,7 +142,7 @@ public class ResourcesUpdater {
         }
 
         executor.shutdown();
-        if (!executor.awaitTermination(10, TimeUnit.MINUTES)) {
+        if (!executor.awaitTermination(5, TimeUnit.HOURS)) {
             throw new RuntimeException("Widgets resources update timeout");
         }
         log.info("Updated {} widgets", updatedCount);

application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationCenter.java

@@ -122,6 +122,7 @@ public class DefaultNotificationCenter extends AbstractSubscriptionService imple
         if (notificationTemplate == null) {
             throw new IllegalArgumentException("Template is missing");
         }
+        NotificationType notificationType = notificationTemplate.getNotificationType();
 
         Set<NotificationDeliveryMethod> deliveryMethods = new HashSet<>();
         List<NotificationTarget> targets = new ArrayList<>();
@@ -143,13 +144,13 @@ public class DefaultNotificationCenter extends AbstractSubscriptionService imple
             try {
                 channels.get(deliveryMethod).check(tenantId);
             } catch (Exception e) {
-                if (ruleId == null) {
+                if (ruleId == null && !notificationType.isSystem()) {
                     throw new IllegalArgumentException(e.getMessage());
                 } else {
-                    return; // if originated by rule - just ignore delivery method
+                    return; // if originated by rule or notification type is system - just ignore delivery method
                 }
             }
-            if (ruleId == null && !notificationTemplate.getNotificationType().isSystem()) {
+            if (ruleId == null && !notificationType.isSystem()) {
                 if (targets.stream().noneMatch(target -> target.getConfiguration().getType().getSupportedDeliveryMethods().contains(deliveryMethod))) {
                     throw new IllegalArgumentException("Recipients for " + deliveryMethod.getName() + " delivery method not chosen");
                 }

application/src/main/resources/thingsboard.yml

@@ -600,7 +600,7 @@ cache:
       maxSize: "${CACHE_SPECS_EDGE_SESSIONS_MAX_SIZE:10000}" # 0 means the cache is disabled
     relatedEdges:
       timeToLiveInMinutes: "${CACHE_SPECS_RELATED_EDGES_TTL:1440}" # Related Edges cache TTL
-      maxSize: "${CACHE_SPECS_RELATED_EDGES_MAX_SIZE:10000}" # 0 means the cache is disabled
+      maxSize: "${CACHE_SPECS_RELATED_EDGES_MAX_SIZE:0}" # 0 means the cache is disabled
     repositorySettings:
       timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" # Repository settings cache TTL
       maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled

application/src/test/java/org/thingsboard/server/cache/CaffeineCacheDefaultConfigurationTest.java

@@ -24,6 +24,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.boot.test.context.SpringBootContextLoader;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -32,6 +33,10 @@ import static org.assertj.core.api.Assertions.assertThat;
 @ContextConfiguration(classes = CaffeineCacheDefaultConfigurationTest.class, loader = SpringBootContextLoader.class)
 @ComponentScan({"org.thingsboard.server.cache"})
 @EnableConfigurationProperties
+@TestPropertySource(properties = {
+        "cache.specs.edgeSessions.timeToLiveInMinutes=1",
+        "cache.specs.relatedEdges.maxSize=1"
+})
 @Slf4j
 public class CaffeineCacheDefaultConfigurationTest {
 
@@ -46,11 +51,7 @@ public class CaffeineCacheDefaultConfigurationTest {
         SoftAssertions softly = new SoftAssertions();
         cacheSpecsMap.getSpecs().forEach((name, cacheSpecs) -> {
             softly.assertThat(name).as("cache name").isNotEmpty();
-            if (name.equals("edgeSessions")) {
-                softly.assertThat(cacheSpecs.getTimeToLiveInMinutes()).as("cache %s time to live", name).isEqualTo(0);
-            } else {
-                softly.assertThat(cacheSpecs.getTimeToLiveInMinutes()).as("cache %s time to live", name).isGreaterThan(0);
-            }
+            softly.assertThat(cacheSpecs.getTimeToLiveInMinutes()).as("cache %s time to live", name).isGreaterThan(0);
             softly.assertThat(cacheSpecs.getMaxSize()).as("cache %s max size", name).isGreaterThan(0);
         });
         softly.assertAll();

application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java

@@ -21,8 +21,10 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.asset.Asset;
 import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.EdgeId;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.dao.service.DaoSqlTest;
@@ -31,6 +33,8 @@ import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg;
 import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
 import org.thingsboard.server.gen.edge.v1.UplinkMsg;
 
+import java.util.UUID;
+
 @DaoSqlTest
 public class RelationEdgeTest extends AbstractEdgeTest {
 
@@ -58,13 +62,14 @@ public class RelationEdgeTest extends AbstractEdgeTest {
 
         // delete relation
         edgeImitator.expectMessageAmount(1);
-        var deletedRelation = doDelete("/api/v2/relation?" +
-                "fromId=" + relation.getFrom().getId().toString() +
-                "&fromType=" + relation.getFrom().getEntityType().name() +
-                "&relationType=" + relation.getType() +
-                "&relationTypeGroup=" + relation.getTypeGroup().name() +
-                "&toId=" + relation.getTo().getId().toString() +
-                "&toType=" + relation.getTo().getEntityType().name(), EntityRelation.class);
+
+        String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&relationTypeGroup=%s&toId=%s&toType=%s",
+                device.getId().toString(), EntityType.DEVICE.name(), "test",
+                RelationTypeGroup.COMMON.name(), asset.getId().toString(), EntityType.ASSET.name()
+        );
+
+        var deletedRelation = doDelete(deleteUrl, EntityRelation.class);
+
         Assert.assertTrue(edgeImitator.waitForMessages());
         latestMessage = edgeImitator.getLatestMessage();
         Assert.assertTrue(latestMessage instanceof RelationUpdateMsg);
@@ -94,13 +99,13 @@ public class RelationEdgeTest extends AbstractEdgeTest {
         edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
         Assert.assertTrue(edgeImitator.waitForResponses());
 
-        EntityRelation relation = doGet("/api/relation?" +
-                "&fromId=" + device2.getUuidId() +
-                "&fromType=" + device2.getId().getEntityType().name() +
-                "&relationType=" + "test" +
-                "&relationTypeGroup=" + RelationTypeGroup.COMMON.name() +
-                "&toId=" + device1.getUuidId() +
-                "&toType=" + device1.getId().getEntityType().name(), EntityRelation.class);
+        String getUrl = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&relationTypeGroup=%s&toId=%s&toType=%s",
+                device2.getUuidId(), EntityType.DEVICE.name(), "test",
+                RelationTypeGroup.COMMON.name(), device1.getUuidId(), EntityType.DEVICE.name()
+        );
+
+        var relation = doGet(getUrl, EntityRelation.class);
+
         Assert.assertNotNull(relation);
     }
 
@@ -154,6 +159,47 @@ public class RelationEdgeTest extends AbstractEdgeTest {
         Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, relationUpdateMsg.getMsgType());
     }
 
+    @Test
+    public void testRelationFromEdgeToDevice() throws Exception {
+        // create relation
+        Device device = findDeviceByName("Edge Device 1");
+        EdgeId edgeId = new EdgeId(new UUID(edgeImitator.getConfiguration().getEdgeIdMSB(), edgeImitator.getConfiguration().getEdgeIdLSB()));
+        EntityRelation relation = new EntityRelation();
+        relation.setType("test");
+        relation.setFrom(edgeId);
+        relation.setTo(device.getId());
+        relation.setTypeGroup(RelationTypeGroup.COMMON);
+        edgeImitator.expectMessageAmount(1);
+        relation = doPost("/api/v2/relation", relation, EntityRelation.class);
+        Assert.assertTrue(edgeImitator.waitForMessages());
+        AbstractMessage latestMessage = edgeImitator.getLatestMessage();
+        Assert.assertTrue(latestMessage instanceof RelationUpdateMsg);
+        RelationUpdateMsg relationUpdateMsg = (RelationUpdateMsg) latestMessage;
+        EntityRelation entityRelation = JacksonUtil.fromString(relationUpdateMsg.getEntity(), EntityRelation.class, true);
+        Assert.assertNotNull(entityRelation);
+        Assert.assertEquals(relation, entityRelation);
+        Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, relationUpdateMsg.getMsgType());
+
+        // delete relation
+        edgeImitator.expectMessageAmount(1);
+
+        String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&relationTypeGroup=%s&toId=%s&toType=%s",
+                edgeId, EntityType.EDGE.name(), "test",
+                RelationTypeGroup.COMMON.name(), device.getId().toString(), EntityType.DEVICE.name()
+        );
+
+        var deletedRelation = doDelete(deleteUrl, EntityRelation.class);
+
+        Assert.assertTrue(edgeImitator.waitForMessages());
+        latestMessage = edgeImitator.getLatestMessage();
+        Assert.assertTrue(latestMessage instanceof RelationUpdateMsg);
+        relationUpdateMsg = (RelationUpdateMsg) latestMessage;
+        entityRelation = JacksonUtil.fromString(relationUpdateMsg.getEntity(), EntityRelation.class, true);
+        Assert.assertNotNull(entityRelation);
+        Assert.assertEquals(deletedRelation, entityRelation);
+        Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, relationUpdateMsg.getMsgType());
+    }
+
     private EntityRelation buildEntityRelationForUplinkMsg(DeviceId deviceId1, DeviceId deviceId2) {
         EntityRelation relation = new EntityRelation();
         relation.setType("test");
@@ -163,4 +209,5 @@ public class RelationEdgeTest extends AbstractEdgeTest {
         relation.setAdditionalInfo(TextNode.valueOf("{}"));
         return relation;
     }
+
 }

application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java

@@ -39,7 +39,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.TimeUnit;
 
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
@@ -50,8 +49,8 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
 
     @Test
     public void testRuleChains() throws Exception {
-        // create rule chain
-        edgeImitator.expectMessageAmount(2);
+        // create rule chain: 2 messages from create rule chain, 2 messages from load metadata
+        edgeImitator.expectMessageAmount(4);
         RuleChain ruleChain = new RuleChain();
         ruleChain.setName("Edge Test Rule Chain");
         ruleChain.setType(RuleChainType.EDGE);
@@ -60,15 +59,21 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
                 + "/ruleChain/" + savedRuleChain.getUuidId(), RuleChain.class);
         createRuleChainMetadata(savedRuleChain);
         Assert.assertTrue(edgeImitator.waitForMessages());
-        Optional<RuleChainUpdateMsg> ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class);
-        Assert.assertTrue(ruleChainUpdateMsgOpt.isPresent());
-        RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgOpt.get();
+        List<RuleChainUpdateMsg> ruleChainUpdateMsgs = edgeImitator.findAllMessagesByType(RuleChainUpdateMsg.class);
+        Assert.assertEquals(2, ruleChainUpdateMsgs.size());
+        List<RuleChainMetadataUpdateMsg> ruleChainMetadataUpdateMsgs = edgeImitator.findAllMessagesByType(RuleChainMetadataUpdateMsg.class);
+        Assert.assertEquals(2, ruleChainMetadataUpdateMsgs.size());
+        RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainUpdateMsgs.get(0);
         RuleChain ruleChainMsg = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true);
         Assert.assertNotNull(ruleChainMsg);
         Assert.assertTrue(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE.equals(ruleChainUpdateMsg.getMsgType()) ||
                 UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE.equals(ruleChainUpdateMsg.getMsgType()));
         Assert.assertEquals(savedRuleChain.getId(), ruleChainMsg.getId());
         Assert.assertEquals(savedRuleChain.getName(), ruleChainMsg.getName());
+        RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateMsgs.get(0);
+        RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true);
+        Assert.assertNotNull(ruleChainMetaData);
+        Assert.assertEquals(ruleChainMetaData.getRuleChainId(), savedRuleChain.getId());
 
         testRuleChainMetadataRequestMsg(savedRuleChain.getId());
 
@@ -77,7 +82,7 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
         doDelete("/api/edge/" + edge.getUuidId()
                 + "/ruleChain/" + savedRuleChain.getUuidId(), RuleChain.class);
         Assert.assertTrue(edgeImitator.waitForMessages());
-        ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class);
+        Optional<RuleChainUpdateMsg> ruleChainUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainUpdateMsg.class);
         Assert.assertTrue(ruleChainUpdateMsgOpt.isPresent());
         ruleChainUpdateMsg = ruleChainUpdateMsgOpt.get();
         Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, ruleChainUpdateMsg.getMsgType());
@@ -262,4 +267,5 @@ public class RuleChainEdgeTest extends AbstractEdgeTest {
                 .andExpect(status().isOk());
         Assert.assertTrue(edgeImitator.waitForMessages(5));
     }
+
 }

application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java

@@ -1697,6 +1697,62 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest {
         assertEquals(4.2d, actual);
     }
 
+    // Bitwise Operations
+    @Test
+    public void bitwiseOperationsBoolean_Test() throws ExecutionException, InterruptedException {
+        msgStr = "{}";
+        decoderStr = """     
+                    var x = true;
+                    var y = false;               
+                    return {
+                        "andResult": x & y,
+                        "orResult": x | y,
+                        "xorResult": x ^ y,                    
+                        "leftShift": x << y,                    
+                        "rightShift": x >> y,                    
+                        "rightUnShift": x >>> y                    
+                    }
+                """;
+        LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
+        expected.put("andResult", 0);
+        expected.put("orResult", 1);
+        expected.put("xorResult", 1);
+        expected.put("leftShift", 1);
+        expected.put("rightShift", 1);
+        expected.put("rightUnShift", 1);
+        Object actual = invokeScript(evalScript(decoderStr), msgStr);
+        assertEquals(expected, actual);
+    }
+    @Test
+    public void bitwiseOperationsMix_Test() throws ExecutionException, InterruptedException {
+        msgStr = "{}";
+        decoderStr = """     
+                    var x = true;
+                    var y = false;
+                    var i = 10;
+                    var b = -14;
+                    var l = 9223372036854775807;            
+                    return {
+                        "andResult": x & b,
+                        "orResult": i | y,
+                        "xorResult": i ^ l,                
+                        "leftShift": l << i,                   
+                        "rightShift": l >> b,                    
+                        "rightUnShift": i >>> x                    
+                    }
+                """;
+        LinkedHashMap<String, Object> expected = new LinkedHashMap<>();
+        expected.put("andResult", 0);
+        expected.put("orResult", 10);
+        expected.put("xorResult", 9223372036854775797L);
+        expected.put("leftShift", -1024L);
+        expected.put("rightShift", 8191L);
+        expected.put("rightUnShift", 5);
+        Object actual = invokeScript(evalScript(decoderStr), msgStr);
+        assertEquals(expected, actual);
+    }
+
+
     // base64
     @Test
     public void base64_Test() throws ExecutionException, InterruptedException {

application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java

@@ -73,13 +73,13 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO
      * @throws Exception
      */
     @Test
-    public void testObserveOneObjectInstance_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception {
+    public void testObserveOneObjectInstance_Result_CONTENT_Value_Count_Equal_Greater_3_After_Cancel_Count_2() throws Exception {
         sendObserveCancelAllWithAwait(lwM2MTestClient.getDeviceIdStr());
         String idVer_3_0 = objectInstanceIdVer_3;
         sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0);
 
         int cntUpdate = 3;
-        verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate))
+        verify(defaultUplinkMsgHandlerTest, timeout(10000).atLeast(cntUpdate))
                 .updateAttrTelemetry(Mockito.any(Registration.class), eq(idVer_3_0_9), eq(null));
     }
 
@@ -88,13 +88,13 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO
      * @throws Exception
      */
     @Test
-    public void testObserveOneObject_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception {
+    public void testObserveOneObject_Result_CONTENT_Value_Count_Equal_Greater_3_After_Cancel_Count_2() throws Exception {
         sendObserveCancelAllWithAwait(lwM2MTestClient.getDeviceIdStr());
         String idVer_3_0 = objectInstanceIdVer_3;
         sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0);
 
         int cntUpdate = 3;
-        verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate))
+        verify(defaultUplinkMsgHandlerTest, timeout(10000).atLeast(cntUpdate))
                 .updateAttrTelemetry(Mockito.any(Registration.class), eq(idVer_3_0_9), eq(null));
     }
 

common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java

@@ -18,6 +18,7 @@ package org.thingsboard.server.dao.resource;
 import com.google.common.util.concurrent.ListenableFuture;
 import org.thingsboard.server.common.data.Dashboard;
 import org.thingsboard.server.common.data.ResourceExportData;
+import org.thingsboard.server.common.data.ResourceSubType;
 import org.thingsboard.server.common.data.ResourceType;
 import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.TbResourceInfo;
@@ -88,6 +89,6 @@ public interface ResourceService extends EntityDaoService {
 
     Collection<TbResourceInfo> getUsedResources(TenantId tenantId, WidgetTypeDetails widgetTypeDetails);
 
-    TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, byte[] data);
+    TbResource createOrUpdateSystemResource(ResourceType resourceType, ResourceSubType resourceSubType, String resourceKey, byte[] data);
 
 }

dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@@ -597,7 +597,7 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
     }
 
     @Override
-    public TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, byte[] data) {
+    public TbResource createOrUpdateSystemResource(ResourceType resourceType, ResourceSubType resourceSubType, String resourceKey, byte[] data) {
         if (resourceType == ResourceType.DASHBOARD) {
             Dashboard dashboard = JacksonUtil.fromBytes(data, Dashboard.class);
             dashboard.setTenantId(TenantId.SYS_TENANT_ID);
@@ -615,6 +615,7 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
             resource = new TbResource();
             resource.setTenantId(TenantId.SYS_TENANT_ID);
             resource.setResourceType(resourceType);
+            resource.setResourceSubType(resourceSubType);
             resource.setResourceKey(resourceKey);
             resource.setFileName(resourceKey);
             resource.setTitle(resourceKey);

dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java

@@ -134,7 +134,7 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao<EdgeEventEnti
 
     @Override
     public ListenableFuture<Void> saveAsync(EdgeEvent edgeEvent) {
-        log.debug("Save edge event [{}] ", edgeEvent);
+        log.debug("Saving EdgeEvent [{}] ", edgeEvent);
         if (edgeEvent.getId() == null) {
             UUID timeBased = Uuids.timeBased();
             edgeEvent.setId(new EdgeEventId(timeBased));
@@ -156,7 +156,7 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao<EdgeEventEnti
     }
 
     private ListenableFuture<Void> save(EdgeEventEntity entity) {
-        log.debug("Save edge event [{}] ", entity);
+        log.debug("Saving EdgeEventEntity [{}] ", entity);
         if (entity.getTenantId() == null) {
             log.trace("Save system edge event with predefined id {}", systemTenantId);
             entity.setTenantId(systemTenantId);

msa/tb/docker/upgrade-tb.sh

@@ -20,23 +20,15 @@ start-db.sh
 CONF_FOLDER="${pkg.installFolder}/conf"
 jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
 configfile=${pkg.name}.conf
-upgradeversion=${DATA_FOLDER}/.upgradeversion
 
 source "${CONF_FOLDER}/${configfile}"
 
-FROM_VERSION=`cat ${upgradeversion}`
-
 echo "Starting ThingsBoard upgrade ..."
 
-fromVersion="${FROM_VERSION// }"
-
 java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \
                 -Dspring.jpa.hibernate.ddl-auto=none \
                 -Dinstall.upgrade=true \
-                -Dinstall.upgrade.from_version=${fromVersion} \
                 -Dlogging.config=/usr/share/thingsboard/bin/install/logback.xml \
                 org.springframework.boot.loader.launch.PropertiesLauncher
 
-echo "${pkg.upgradeVersion}" > ${upgradeversion}
-
 stop-db.sh

ui-ngx/.browserslistrc

@@ -0,0 +1,19 @@
+# This file is used by the build system to adjust CSS and JS output to support the specified browsers below.
+# For additional information regarding the format and rule options, please see:
+# https://github.com/browserslist/browserslist#queries
+
+# For the full list of supported browsers by the Angular framework, please see:
+# https://angular.dev/reference/versions#browser-support
+
+# You can see what browsers were selected by your queries by running:
+#   npx browserslist
+
+>0.5%
+not dead
+last 2 Chrome versions
+last 1 Firefox version
+last 2 Edge major versions
+last 2 Safari major versions
+last 2 iOS major versions
+last 2 Android major versions
+Firefox ESR

ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts

@@ -52,6 +52,7 @@ import {
   isUndefined,
   isUndefinedOrNull,
   mergeDeep,
+  mergeDeepIgnoreArray,
   parseFunction
 } from '@core/utils';
 import { BehaviorSubject, forkJoin, Observable, Observer, of, Subject } from 'rxjs';
@@ -516,7 +517,7 @@ export class ScadaSymbolObject {
     const doc: XMLDocument = new DOMParser().parseFromString(this.svgContent, 'image/svg+xml');
     this.metadata = parseScadaSymbolMetadataFromDom(doc);
     const defaults = defaultScadaSymbolObjectSettings(this.metadata);
-    this.settings = mergeDeep<ScadaSymbolObjectSettings>({} as ScadaSymbolObjectSettings,
+    this.settings = mergeDeepIgnoreArray<ScadaSymbolObjectSettings>({} as ScadaSymbolObjectSettings,
       defaults, this.inputSettings || {} as ScadaSymbolObjectSettings);
     this.prepareMetadata();
     this.prepareSvgShape(doc);

ui-ngx/src/app/modules/home/components/widget/lib/settings/common/scada/scada-symbol-object-settings.component.ts

@@ -44,7 +44,7 @@ import {
 } from '@home/components/widget/lib/scada/scada-symbol.models';
 import { IAliasController } from '@core/api/widget-api.models';
 import { TargetDevice, widgetType } from '@shared/models/widget.models';
-import { mergeDeep } from '@core/utils';
+import { isDefinedAndNotNull, mergeDeepIgnoreArray } from '@core/utils';
 import {
   ScadaSymbolBehaviorGroup,
   toBehaviorGroups
@@ -207,7 +207,7 @@ export class ScadaSymbolObjectSettingsComponent implements OnInit, OnChanges, Co
   private setupValue() {
     if (this.metadata) {
       const defaults = defaultScadaSymbolObjectSettings(this.metadata);
-      this.modelValue = mergeDeep<ScadaSymbolObjectSettings>(defaults, this.modelValue);
+      this.modelValue = mergeDeepIgnoreArray<ScadaSymbolObjectSettings>(defaults, this.modelValue);
       this.scadaSymbolObjectSettingsFormGroup.patchValue(
         this.modelValue, {emitEvent: false}
       );

ui-ngx/src/app/modules/home/pages/admin/oauth2/clients/clients-table-config.resolver.ts

@@ -20,15 +20,10 @@ import {
   EntityTableColumn,
   EntityTableConfig
 } from '@home/models/entity/entities-table-config.models';
-import {
-  getProviderHelpLink,
-  OAuth2Client,
-  OAuth2ClientInfo,
-  platformTypeTranslations
-} from '@shared/models/oauth2.models';
+import { OAuth2Client, OAuth2ClientInfo, platformTypeTranslations } from '@shared/models/oauth2.models';
 import { TranslateService } from '@ngx-translate/core';
 import { DatePipe } from '@angular/common';
-import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models';
+import { EntityType, entityTypeResources, entityTypeTranslations } from '@shared/models/entity-type.models';
 import { OAuth2Service } from '@core/http/oauth2.service';
 import { ClientComponent } from '@home/pages/admin/oauth2/clients/client.component';
 import { ClientTableHeaderComponent } from '@home/pages/admin/oauth2/clients/client-table-header.component';
@@ -48,10 +43,7 @@ export class ClientsTableConfigResolver  {
     this.config.entityType = EntityType.OAUTH2_CLIENT;
     this.config.rowPointer = true;
     this.config.entityTranslations = entityTypeTranslations.get(EntityType.OAUTH2_CLIENT);
-    this.config.entityResources = {
-      helpLinkId: null,
-      helpLinkIdForEntity: (entity: OAuth2Client) => getProviderHelpLink(entity.additionalInfo.providerName)
-    };
+    this.config.entityResources = entityTypeResources.get(EntityType.OAUTH2_CLIENT);
     this.config.entityComponent = ClientComponent;
     this.config.headerComponent = ClientTableHeaderComponent;
     this.config.addDialogStyle = {width: '850px', maxHeight: '100vh'};

ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts

@@ -63,7 +63,9 @@ export class EditorPanelComponent implements OnInit {
         const uiContainer = document.querySelector('.tox.tox-tinymce-aux');
         container.parentNode.appendChild(uiContainer);
       });
-    }
+    },
+    relative_urls: false,
+    urlconverter_callback: (url) => url
   };
 
   constructor(private fb: FormBuilder) {

ui-ngx/src/app/modules/home/pages/notification/template/configuration/notification-template-configuration.component.ts

@@ -95,7 +95,8 @@ export class NotificationTemplateConfigurationComponent implements OnDestroy, Co
     autofocus: false,
     branding: false,
     promotion: false,
-    relative_urls: false
+    relative_urls: false,
+    urlconverter_callback: (url) => url
   };
 
   private propagateChange = null;

ui-ngx/src/app/shared/components/js-func-module-row.component.html

@@ -18,6 +18,14 @@
 <div [formGroup]="moduleRowFormGroup" class="tb-form-table-row tb-js-func-module-row">
   <mat-form-field class="tb-inline-field tb-alias-field" appearance="outline" subscriptSizing="dynamic">
     <input required matInput formControlName="alias" placeholder="{{ 'widget-config.set' | translate }}">
+    <mat-icon matSuffix
+              matTooltipPosition="above"
+              matTooltipClass="tb-error-tooltip"
+              [matTooltip]="'js-func.invalid-module-alias-name' | translate"
+              *ngIf="moduleRowFormGroup.get('alias').invalid && moduleRowFormGroup.get('alias').touched"
+              class="tb-error">
+      warning
+    </mat-icon>
   </mat-form-field>
   <tb-resource-autocomplete class="tb-module-link-field"
                             #resourceAutocomplete

ui-ngx/src/app/shared/components/js-func-module-row.component.ts

@@ -39,7 +39,7 @@ import {
 } from '@angular/forms';
 import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component';
 import { ResourceSubType } from '@shared/models/resource.models';
-import { Observable, of } from 'rxjs';
+import { Observable } from 'rxjs';
 import { ResourceAutocompleteComponent } from '@shared/components/resource/resource-autocomplete.component';
 import { HttpClient } from '@angular/common/http';
 import { loadModuleMarkdownDescription, loadModuleMarkdownSourceCode } from '@shared/models/js-function.models';
@@ -103,7 +103,7 @@ export class JsFuncModuleRowComponent implements ControlValueAccessor, OnInit, V
 
   ngOnInit() {
     this.moduleRowFormGroup = this.fb.group({
-      alias: [null, [this.moduleAliasValidator()]],
+      alias: [null, [this.moduleAliasValidator(), Validators.pattern(/^[$_\p{ID_Start}][$\p{ID_Continue}]*$/u)]],
       moduleLink: [null, [Validators.required]]
     });
     this.moduleRowFormGroup.valueChanges.pipe(
@@ -133,7 +133,7 @@ export class JsFuncModuleRowComponent implements ControlValueAccessor, OnInit, V
 
   public validate(_c: UntypedFormControl) {
     const aliasControl = this.moduleRowFormGroup.get('alias');
-    if (aliasControl.hasError('moduleAliasNotUnique')) {
+    if (aliasControl.hasError('moduleAliasNotUnique') || aliasControl.hasError('pattern')) {
       aliasControl.updateValueAndValidity({onlySelf: false, emitEvent: false});
     }
     if (aliasControl.hasError('moduleAliasNotUnique')) {
@@ -142,6 +142,12 @@ export class JsFuncModuleRowComponent implements ControlValueAccessor, OnInit, V
         moduleAliasNotUnique: true
       };
     }
+    if (aliasControl.hasError('pattern')) {
+      this.moduleRowFormGroup.get('alias').markAsTouched();
+      return {
+        invalidVariableName: true
+      };
+    }
     const module: JsFuncModuleRow = {...this.modelValue, ...this.moduleRowFormGroup.value};
     if (!moduleValid(module)) {
       return {

ui-ngx/src/app/shared/components/js-func-modules.component.ts

@@ -119,6 +119,7 @@ export class JsFuncModulesComponent implements OnInit {
 
   removeModule(index: number, emitEvent = true) {
     (this.modulesFormGroup.get('modules') as UntypedFormArray).removeAt(index, {emitEvent});
+    this.modulesFormGroup.get('modules').markAsDirty({emitEvent});
   }
 
   addModule() {

ui-ngx/src/app/shared/models/entity-type.models.ts

@@ -16,6 +16,7 @@
 
 import { TenantId } from './id/tenant-id';
 import { BaseData, HasId } from '@shared/models/base-data';
+import { getProviderHelpLink, OAuth2Client } from '@shared/models/oauth2.models';
 
 export enum EntityType {
   TENANT = 'TENANT',
@@ -588,7 +589,8 @@ export const entityTypeResources = new Map<EntityType, EntityTypeResource<BaseDa
     [
       EntityType.OAUTH2_CLIENT,
       {
-        helpLinkId: 'oauth2Settings'
+        helpLinkId: 'oauth2Settings',
+        helpLinkIdForEntity: (entity: OAuth2Client) => getProviderHelpLink(entity.additionalInfo.providerName)
       }
     ],
     [

ui-ngx/src/app/shared/models/js-function.models.ts

@@ -98,7 +98,7 @@ export const loadModuleMarkdownDescription = (http: HttpClient, translate: Trans
         } else {
           propDescription += `<p class="!pl-4 !pr-4"><em>const</em> <strong>${propName}</strong>: <code>${type}</code>`;
           if (type !== 'object') {
-            propDescription += ` = ${prop}`;
+            propDescription += ' = ' + (type === 'string' ? `"${handleHtmlSpecialChars(prop)}"` : `${prop}`);
           }
           propDescription += '</p>';
         }
@@ -133,6 +133,10 @@ export const loadModuleMarkdownDescription = (http: HttpClient, translate: Trans
   );
 }
 
+const handleHtmlSpecialChars = (text: string): string => {
+  return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
 export const loadModuleMarkdownSourceCode = (http: HttpClient, translate: TranslateService, resource: ResourceInfo): Observable<string> => {
   let sourceCode = `<div class="flex flex-col !pl-4"><h6>${resource.title}</h6><small>${translate.instant('js-func.source-code')}</small></div>\n\n`;
   return loadFunctionModuleSource(http, resource.link).pipe(

ui-ngx/src/assets/locale/locale.constant-en_US.json

@@ -3377,6 +3377,7 @@
         "no-modules": "No modules configured",
         "add-module": "Add module",
         "module-alias": "Alias",
+        "invalid-module-alias-name": "Invalid alias name",
         "module-resource": "JS module resource",
         "not-unique-module-aliases-error": "Modules aliases must be unique!",
         "show-module-info": "Show module info",