Merge branch 'rc-4.0.2' into release-4.0

application/src/main/data/upgrade/basic/schema_update.sql

@@ -14,82 +14,3 @@
 -- limitations under the License.
 --
 
--- UPDATE SAVE TIME SERIES NODES START
-
-UPDATE rule_node
-SET configuration = (
-    (configuration::jsonb - 'skipLatestPersistence')
-        || jsonb_build_object(
-            'processingSettings', jsonb_build_object(
-                    'type',       'ADVANCED',
-                    'timeseries',       jsonb_build_object('type', 'ON_EVERY_MESSAGE'),
-                    'latest',           jsonb_build_object('type', 'SKIP'),
-                    'webSockets',       jsonb_build_object('type', 'ON_EVERY_MESSAGE'),
-                    'calculatedFields', jsonb_build_object('type', 'ON_EVERY_MESSAGE')
-                                  )
-           )
-    )::text,
-    configuration_version = 1
-WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'
-  AND configuration_version = 0
-  AND configuration::jsonb ->> 'skipLatestPersistence' = 'true';
-
-UPDATE rule_node
-SET configuration = (
-    (configuration::jsonb - 'skipLatestPersistence')
-        || jsonb_build_object(
-            'processingSettings', jsonb_build_object(
-                    'type', 'ON_EVERY_MESSAGE'
-                                  )
-           )
-    )::text,
-    configuration_version = 1
-WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'
-  AND configuration_version = 0
-  AND (configuration::jsonb ->> 'skipLatestPersistence' != 'true' OR configuration::jsonb ->> 'skipLatestPersistence' IS NULL);
-
--- UPDATE SAVE TIME SERIES NODES END
-
--- UPDATE SAVE ATTRIBUTES NODES START
-
-UPDATE rule_node
-SET configuration = (
-    configuration::jsonb
-        || jsonb_build_object(
-            'processingSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE')
-           )
-    )::text,
-    configuration_version = 3
-WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode'
-  AND configuration_version = 2;
-
--- UPDATE SAVE ATTRIBUTES NODES END
-
-ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1;
-
--- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START
-
-UPDATE tenant_profile
-SET profile_data = profile_data
-    || jsonb_build_object(
-                           'configuration', profile_data->'configuration' || jsonb_build_object(
-                    'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint,
-                    'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint,
-                    'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint,
-                    'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint,
-                    'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint
-                                                                             )
-       )
-WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL;
-
--- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END
-
--- UPDATE TENANT PROFILE DEBUG DURATION START
-
-UPDATE tenant_profile
-SET profile_data = jsonb_set(profile_data, '{configuration,maxDebugModeDurationMinutes}', '15', true)
-WHERE
-    profile_data->'configuration' ? 'maxDebugModeDurationMinutes' = false
-    OR (profile_data->'configuration'->>'maxDebugModeDurationMinutes')::int = 0;
-
--- UPDATE TENANT PROFILE DEBUG DURATION END

application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java

@@ -328,6 +328,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
         Edge edge = edgeGrpcSession.getEdge();
         TenantId tenantId = edge.getTenantId();
         log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId);
+        if (sessions.containsKey(edgeId)) {
+            destroySession(sessions.get(edgeId));
+        }
         sessions.put(edgeId, edgeGrpcSession);
         final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock());
         newEventLock.lock();
@@ -493,7 +496,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
             } finally {
                 newEventLock.unlock();
             }
-            toRemove.destroy();
+            destroySession(toRemove);
             TenantId tenantId = toRemove.getEdge().getTenantId();
             save(tenantId, edgeId, ACTIVITY_STATE, false);
             long lastDisconnectTs = System.currentTimeMillis();
@@ -506,6 +509,12 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
         edgeIdServiceIdCache.evict(edgeId);
     }
 
+    private void destroySession(EdgeGrpcSession session) {
+        try (session) {
+            session.destroy();
+        }
+    }
+
     private void save(TenantId tenantId, EdgeId edgeId, String key, long value) {
         log.debug("[{}][{}] Updating long edge telemetry [{}] [{}]", tenantId, edgeId, key, value);
         if (persistToTelemetry) {

application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java

@@ -32,7 +32,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti
 
     // This list should include all versions which are compatible for the upgrade.
     // The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release.
-    private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.9.0", "3.9.1", "4.0.0");
+    private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("4.0.0", "4.0.1");
 
     private final ProjectInfo projectInfo;
     private final JdbcTemplate jdbcTemplate;

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java

@@ -36,7 +36,7 @@ public abstract class AbstractEntityProfileNameQueryProcessor<T extends EntityFi
     public AbstractEntityProfileNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) {
         super(repo, ctx, query, filter, entityType);
         entityProfileNames = new HashSet<>(getProfileNames(this.filter));
-        pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter));
+        pattern = RepositoryUtils.toEntityNameSqlLikePattern(getEntityNameFilter(filter));
     }
 
     protected abstract String getEntityNameFilter(T filter);

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java

@@ -43,7 +43,7 @@ public abstract class AbstractEntityProfileQueryProcessor<T extends EntityFilter
                 entityProfileIds.add(dp.getId());
             }
         }
-        pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter));
+        pattern = RepositoryUtils.toEntityNameSqlLikePattern(getEntityNameFilter(filter));
     }
 
     protected abstract String getEntityNameFilter(T filter);

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.edqs.query.processor;
 
+import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 import org.thingsboard.server.common.data.permission.QueryContext;
 import org.thingsboard.server.common.data.query.EntityFilter;
@@ -106,7 +107,7 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
 
     private Set<EntityData<?>> getEntitiesSet(RelationsRepo relations) {
         Set<EntityData<?>> result = new HashSet<>();
-        Set<UUID> processed = new HashSet<>();
+        Set<RelationSearchTask> processed = new HashSet<>();
         Queue<RelationSearchTask> tasks = new LinkedList<>();
         int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel());
         for (UUID uuid : getRootEntities()) {
@@ -114,7 +115,7 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
         }
         while (!tasks.isEmpty()) {
             RelationSearchTask task = tasks.poll();
-            if (processed.add(task.entityId)) {
+            if (processed.add(task)) {
                 var entityLvl = task.lvl + 1;
                 Set<RelationInfo> entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId);
                 if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) {
@@ -157,6 +158,7 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
     protected abstract boolean check(RelationInfo relationInfo);
 
     @RequiredArgsConstructor
+    @EqualsAndHashCode
     private static class RelationSearchTask {
         private final UUID entityId;
         private final int lvl;

common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java

@@ -30,7 +30,7 @@ public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor<Entit
 
     public EntityNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
         super(repo, ctx, query, (EntityNameFilter) query.getEntityFilter(), ((EntityNameFilter) query.getEntityFilter()).getEntityType());
-        pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityNameFilter());
+        pattern = RepositoryUtils.toEntityNameSqlLikePattern(filter.getEntityNameFilter());
     }
 
     @Override

common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java

@@ -54,7 +54,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
@@ -249,11 +248,11 @@ public class RepositoryUtils {
         }
         return switch (predicate.getOperation()) {
             case EQUAL -> value.equals(predicateValue);
-            case STARTS_WITH -> value.startsWith(predicateValue);
+            case STARTS_WITH -> toSqlLikePattern(predicateValue, "^", ".*").matcher(value).matches();
-            case ENDS_WITH -> value.endsWith(predicateValue);
+            case ENDS_WITH -> toSqlLikePattern(predicateValue, ".*", "$").matcher(value).matches();
             case NOT_EQUAL -> !value.equals(predicateValue);
-            case CONTAINS -> value.contains(predicateValue);
+            case CONTAINS -> toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches();
-            case NOT_CONTAINS -> !value.contains(predicateValue);
+            case NOT_CONTAINS -> !toSqlLikePattern(predicateValue, ".*", ".*").matcher(value).matches();
             case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue));
             case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue));
         };
@@ -304,6 +303,15 @@ public class RepositoryUtils {
             return true;
         } else if (filterPredicates.getOperation() == OR) {
             for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) {
+
+                // Emulate the SQL-like behavior of ThingsBoard's Entity Data Query service:
+                // for COMPLEX filters, return no results if filter value is empty
+                if (filterPredicate instanceof StringFilterPredicate stringFilterPredicate) {
+                    if (StringUtils.isEmpty(stringFilterPredicate.getValue().getValue())) {
+                        continue;
+                    }
+                }
+
                 if (simpleKeyFilter.check(value, filterPredicate)) {
                     return true;
                 }
@@ -314,25 +322,34 @@ public class RepositoryUtils {
         }
     }
 
-    public static Pattern toSqlLikePattern(String nameFilter) {
+    public static Pattern toEntityNameSqlLikePattern(String filter) {
-        if (StringUtils.isNotBlank(nameFilter)) {
+        if (StringUtils.isNotBlank(filter)) {
-            boolean percentSymbolOnStart = nameFilter.startsWith("%");
+            return toSqlLikePattern(filter, "", ".*", true);
-            boolean percentSymbolOnEnd = nameFilter.endsWith("%");
-            if (percentSymbolOnStart) {
-                nameFilter = nameFilter.substring(1);
-            }
-            if (percentSymbolOnEnd) {
-                nameFilter = nameFilter.substring(0, nameFilter.length() - 1);
-            }
-            if (percentSymbolOnStart || percentSymbolOnEnd) {
-                return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE);
-            } else {
-                return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE);
-            }
         }
         return null;
     }
 
+    private static Pattern toSqlLikePattern(String value, String prefix, String suffix) {
+        return toSqlLikePattern(value, prefix, suffix, false);
+    }
+
+    private static Pattern toSqlLikePattern(String value, String prefix, String suffix, boolean ignoreCase) {
+        String regexValue;
+        if (value.contains("%") || value.contains("_")) {
+            regexValue = value
+                    .replace("_", ".")
+                    .replace("%", ".*");
+            if ("^".equals(prefix)) {
+                regexValue = "^" + regexValue + (regexValue.endsWith(".*") ? "" : ".*");
+            } else if ("$".equals(suffix)) {
+                regexValue = (regexValue.startsWith(".*") ? "" : ".*") + regexValue + "$";
+            }
+        } else {
+            regexValue = prefix + Pattern.quote(value) + suffix;
+        }
+        return ignoreCase ? Pattern.compile(regexValue, Pattern.CASE_INSENSITIVE) : Pattern.compile(regexValue);
+    }
+
     @FunctionalInterface
     public interface SimpleKeyFilter<T> {
 

edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java

@@ -70,7 +70,45 @@ public class RepositoryUtilsTest {
                 Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true),
                 Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false),
                 Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true),
-                Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false)
+                Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false),
+
+                // Basic CONTAINS
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranet"), false),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%ranet%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%123"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%loranx%"), false),
+
+                // Basic STARTS_WITH
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lorax%"), false),
+
+                // Basic ENDS_WITH
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%123"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%23"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "%124"), false),
+
+                // CONTAINS with _
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_123"), true), // '_' = ' '
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet_12_"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loran_t%"), true),
+
+                // STARTS_WITH with _
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "loranet_"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lora__t%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "lor_net%"), true),
+
+                // ENDS_WITH with _
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_23"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_2_"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "_3"), true),
+
+                // Mixed patterns
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora__t 1%"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora%net%3"), true),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "%o_anet%2_3"), false),
+                Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "lora___ ___"), true)
         );
     }