diff --git a/TEST_FAST.md b/TEST_FAST.md index eb2013c601..fbea72db9c 100644 --- a/TEST_FAST.md +++ b/TEST_FAST.md @@ -10,6 +10,7 @@ export SUREFIRE_JAVA_OPTS="-Xmx1200m -Xss256k -XX:+ExitOnOutOfMemoryError" mvn clean install -T6 -DskipTests -Dpkg.skip=true mvn test -pl='!application,!dao,!ui-ngx,!msa/js-executor,!msa/web-ui' -T4 +mvn test -pl='msa/js-executor' mvn test -pl dao -Dparallel=packages -DforkCount=4 mvn test -pl application -Dtest='!**/nosql/**,org.thingsboard.server.controller.**' -DforkCount=6 -Dparallel=classes -Dsurefire.rerunFailingTestsCount=2 -Dsurefire.failOnFlakeCount=5 diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 96f58707b5..aaf066acdc 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -135,6 +135,10 @@ public class InstallScripts { return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR); } + public Path getWidgetBundlesDir() { + return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); + } + public String getDataDir() { if (!StringUtils.isEmpty(dataDir)) { if (!Paths.get(this.dataDir).toFile().isDirectory()) { @@ -207,7 +211,7 @@ public class InstallScripts { public void loadSystemWidgets() { log.info("Loading system widgets"); Map widgetsBundlesMap = new HashMap<>(); - Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); + Path widgetBundlesDir = getWidgetBundlesDir(); try (Stream dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(JSON_EXT))) { dirStream.forEach( path -> { diff --git a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java index bb5052fd76..a256888107 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java +++ b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java @@ -28,8 +28,10 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.DatabaseSchemaSettingsService; import org.thingsboard.server.service.install.InstallScripts; @@ -42,6 +44,9 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -67,6 +72,7 @@ public class SystemPatchApplier { private final InstallScripts installScripts; private final DatabaseSchemaSettingsService schemaSettingsService; private final WidgetTypeService widgetTypeService; + private final WidgetsBundleService widgetsBundleService; private final ImageService imageService; @PostConstruct @@ -100,8 +106,11 @@ public class SystemPatchApplier { updateSqlViews(); log.info("Updated sql database views"); - int updated = updateWidgetTypes(); - log.info("Updated {} widget types", updated); + WidgetTypeStats widgetStats = updateWidgetTypes(); + log.info("System widget types: {} created, {} updated", widgetStats.created(), widgetStats.updated()); + + int updatedBundles = updateWidgetBundles(); + log.info("System widget bundles: {} updated", updatedBundles); int createdImages = createMissingSystemImages(); log.info("Created {} new system images", createdImages); @@ -171,20 +180,24 @@ public class SystemPatchApplier { } } - private int updateWidgetTypes() { + private WidgetTypeStats updateWidgetTypes() { + AtomicInteger created = new AtomicInteger(); AtomicInteger updated = new AtomicInteger(); Path widgetTypesDir = installScripts.getWidgetTypesDir(); if (!Files.exists(widgetTypesDir)) { log.trace("Widget types directory does not exist: {}", widgetTypesDir); - return 0; + return new WidgetTypeStats(0, 0); } try (Stream dirStream = listDir(widgetTypesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) { dirStream.forEach( path -> { try { - if (updateWidgetTypeFromFile(path)) { + WidgetTypeChange change = updateWidgetTypeFromFile(path); + if (change == WidgetTypeChange.CREATED) { + created.incrementAndGet(); + } else if (change == WidgetTypeChange.UPDATED) { updated.incrementAndGet(); } } catch (Exception e) { @@ -195,18 +208,26 @@ public class SystemPatchApplier { ); } - return updated.get(); + return new WidgetTypeStats(created.get(), updated.get()); } - private boolean updateWidgetTypeFromFile(Path filePath) { + private WidgetTypeChange updateWidgetTypeFromFile(Path filePath) { JsonNode json = JacksonUtil.toJsonNode(filePath.toFile()); WidgetTypeDetails fileWidgetType = JacksonUtil.treeToValue(json, WidgetTypeDetails.class); + return saveOrUpdateSystemWidgetType(fileWidgetType); + } + + private WidgetTypeChange saveOrUpdateSystemWidgetType(WidgetTypeDetails fileWidgetType) { String fqn = fileWidgetType.getFqn(); + if (fqn == null || fqn.isBlank()) { + throw new RuntimeException("Widget type fqn is missing or blank: " + fileWidgetType.getName()); + } WidgetTypeDetails existingWidgetType = widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, fqn); if (existingWidgetType == null) { - // We expect only update here, so it's probably never happening, but for test purpose leave it like this: - throw new RuntimeException("Widget type not found: " + fqn); + widgetTypeService.saveWidgetType(fileWidgetType); + log.trace("Created widget type: {}", fqn); + return WidgetTypeChange.CREATED; } if (isWidgetTypeChanged(existingWidgetType, fileWidgetType)) { existingWidgetType.setDescription(fileWidgetType.getDescription()); @@ -214,11 +235,95 @@ public class SystemPatchApplier { existingWidgetType.setDescriptor(fileWidgetType.getDescriptor()); widgetTypeService.saveWidgetType(existingWidgetType); log.trace("Updated widget type: {}", fqn); - return true; + return WidgetTypeChange.UPDATED; } log.trace("Widget type unchanged: {}", fqn); - return false; + return WidgetTypeChange.UNCHANGED; + } + + private int updateWidgetBundles() { + AtomicInteger updated = new AtomicInteger(); + Path widgetBundlesDir = installScripts.getWidgetBundlesDir(); + + if (!Files.exists(widgetBundlesDir)) { + log.trace("Widget bundles directory does not exist: {}", widgetBundlesDir); + return 0; + } + + try (Stream dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) { + dirStream.forEach(path -> { + try { + if (processWidgetBundleFile(path)) { + updated.incrementAndGet(); + } + } catch (Exception e) { + log.error("Unable to process widgets bundle from json: [{}]", path); + throw new RuntimeException("Unable to process widgets bundle from json", e); + } + }); + } + + return updated.get(); + } + + private boolean processWidgetBundleFile(Path filePath) { + JsonNode bundleJson = JacksonUtil.toJsonNode(filePath.toFile()); + if (bundleJson == null || !bundleJson.has("widgetsBundle")) { + throw new RuntimeException("Invalid widgets bundle json: " + filePath); + } + WidgetsBundle fileBundle = JacksonUtil.treeToValue(bundleJson.get("widgetsBundle"), WidgetsBundle.class); + String alias = fileBundle.getAlias(); + if (alias == null || alias.isBlank()) { + throw new RuntimeException("Widgets bundle alias is missing or blank: " + filePath); + } + + WidgetsBundle existingBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, alias); + if (existingBundle == null) { + log.warn("Widgets bundle '{}' not found in DB; bundle creation is not supported by the patch applier, skipping.", alias); + return false; + } + + if (bundleJson.has("widgetTypes")) { + throw new RuntimeException("Inline widgetTypes in bundle JSON are not supported by the patch applier; " + + "place widget definitions in widget_types/*.json and reference them via widgetTypeFqns: " + filePath); + } + List fileWidgetFqns = new ArrayList<>(); + if (bundleJson.has("widgetTypeFqns")) { + bundleJson.get("widgetTypeFqns").forEach(fqnJson -> fileWidgetFqns.add(fqnJson.asText())); + } + + boolean changed = false; + if (isWidgetsBundleChanged(existingBundle, fileBundle)) { + existingBundle.setTitle(fileBundle.getTitle()); + existingBundle.setDescription(fileBundle.getDescription()); + existingBundle.setImage(fileBundle.getImage()); + existingBundle.setOrder(fileBundle.getOrder()); + existingBundle.setScada(fileBundle.isScada()); + widgetsBundleService.saveWidgetsBundle(existingBundle); + log.trace("Updated widgets bundle metadata: {}", alias); + changed = true; + } + + List existingFqns = widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()); + LinkedHashSet mergedFqns = new LinkedHashSet<>(existingFqns); + if (mergedFqns.addAll(fileWidgetFqns)) { + widgetTypeService.updateWidgetsBundleWidgetFqns(TenantId.SYS_TENANT_ID, existingBundle.getId(), new ArrayList<>(mergedFqns)); + log.trace("Linked {} new widget fqn(s) to bundle: {}", mergedFqns.size() - existingFqns.size(), alias); + changed = true; + } + return changed; + } + + private boolean isWidgetsBundleChanged(WidgetsBundle existing, WidgetsBundle file) { + // Image is intentionally NOT compared: the file always carries a base64 data URI, while the DB stores + // the system-image URL produced by ImageService.replaceBase64WithImageUrl on save. A naive string compare + // would always report a diff and re-save every system bundle on every patch run. Image content changes + // are out of scope for the patch applier — full reinstall covers them. + return !Objects.equals(existing.getTitle(), file.getTitle()) + || !Objects.equals(existing.getDescription(), file.getDescription()) + || !Objects.equals(existing.getOrder(), file.getOrder()) + || existing.isScada() != file.isScada(); } private int createMissingSystemImages() { @@ -349,4 +454,8 @@ public class SystemPatchApplier { public record VersionInfo(int major, int minor, int maintenance, int patch) {} + public record WidgetTypeStats(int created, int updated) {} + + private enum WidgetTypeChange { CREATED, UPDATED, UNCHANGED } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2a9e632a05..b1e559fb1a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -465,6 +465,7 @@ sql: log_tenant_stats: "${SQL_LOG_TENANT_STATS:true}" # Interval in milliseconds for printing the latest statistic information about the tenant log_tenant_stats_interval_ms: "${SQL_LOG_TENANT_STATS_INTERVAL_MS:60000}" + entity_data_query_nulls_order_strategy: "${SQL_ENTITY_DATA_QUERY_NULLS_ORDER_STRATEGY:default}" # Nulls ordering strategy for sql entity data query. Possible values: default, nulls_first, nulls_last. The default value is 'default', which means postgres default behavior will be applied: NULLS LAST for ASC and NULLS FIRST for DESC. The 'nulls_first' value means that NULL values will be ordered before non-NULL values regardless of the sort order. The 'nulls_last' value means that NULL values will be ordered after non-NULL values regardless of the sort order. postgres: # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" @@ -1715,8 +1716,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java index d53e69d721..bcffd9099e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java @@ -448,4 +448,21 @@ public class RuleChainControllerTest extends AbstractControllerTest { return doPost("/api/ruleChain", ruleChain, RuleChain.class); } + @Test + public void testScriptForbiddenForCustomer() throws Exception { + loginCustomerUser(); + + doPost("/api/ruleChain/testScript", (Object) """ + { + "script": "return msg;", + "scriptType": "update", + "argNames": ["msg", "metadata", "msgType"], + "msg": "{}", + "metadata": {}, + "msgType": "POST_TELEMETRY_REQUEST" + } + """) + .andExpect(status().isForbidden()); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index adf664e7cf..eb672a790f 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -16,19 +16,27 @@ package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.IdBased; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -43,10 +51,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @DaoSqlTest @TestPropertySource(properties = { @@ -103,6 +114,100 @@ public class EdqsEntityServiceTest extends EntityServiceTest { assetService.deleteAssetsByTenantId(tenantId); } + // edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC + @Override + @Test + public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(1L, 0L, 0L); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "0", "0", "1")); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).containsExactlyElementsOf(List.of("1", "0", "0", "", "")); + } + + // edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC + @Override + @Test + public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(true, false, false); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "false", "false", "true")); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).containsExactlyElementsOf(List.of("true", "false", "false", "", "")); + } + @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 05ab1165f6..b3230b3d45 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; @@ -47,6 +48,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; @@ -79,7 +81,6 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; -import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; @@ -100,6 +101,7 @@ import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -124,7 +126,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.common.data.AttributeScope.SERVER_SCOPE; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; -import static org.thingsboard.server.common.data.query.EntityKeyType.SERVER_ATTRIBUTE; @Slf4j @DaoSqlTest @@ -154,6 +155,8 @@ public class EntityServiceTest extends AbstractControllerTest { @Autowired RelationRepository relationRepository; @Autowired + DefaultEntityQueryRepository entityQueryRepository; + @Autowired RelationService relationService; @Autowired TimeseriesService timeseriesService; @@ -1750,6 +1753,117 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + @Test + public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + try { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(1L, 0L, 0L); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + assertNullsOrdering("default", + List.of("0", "0", "1", "", ""), + List.of("", "", "1", "0", "0"), + devices.size()); + + assertNullsOrdering("nulls_first", + List.of("", "", "0", "0", "1"), + List.of("", "", "1", "0", "0"), + devices.size()); + + assertNullsOrdering("nulls_last", + List.of("0", "0", "1", "", ""), + List.of("1", "0", "0", "", ""), + devices.size()); + } finally { + deviceService.deleteDevicesByTenantId(tenantId); + } + } + + @Test + public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + try { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(true, false, false); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + assertNullsOrdering("default", + List.of("false", "false", "true", "", ""), + List.of("", "", "true", "false", "false"), + devices.size()); + + assertNullsOrdering("nulls_first", + List.of("", "", "false", "false", "true"), + List.of("", "", "true", "false", "false"), + devices.size()); + + assertNullsOrdering("nulls_last", + List.of("false", "false", "true", "", ""), + List.of("true", "false", "false", "", ""), + devices.size()); + } finally { + deviceService.deleteDevicesByTenantId(tenantId); + } + } + + private void assertNullsOrdering(String strategy, List expectedAsc, List expectedDesc, int deviceSize) { + String originalStrategy = entityQueryRepository.getNullsOrderStrategy(); + ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", strategy); + try { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, deviceSize).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).as("ASC with strategy '%s'", strategy).containsExactlyElementsOf(expectedAsc); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, deviceSize).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).as("DESC with strategy '%s'", strategy).containsExactlyElementsOf(expectedDesc); + } finally { + ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", originalStrategy); + } + } + @Test public void testFindTenantTelemetry() throws ExecutionException, InterruptedException, TimeoutException { // save timeseries by sys admin @@ -2321,12 +2435,18 @@ public class EntityServiceTest extends AbstractControllerTest { return timeseriesService.save(tenantId, entityId, timeseries); } - private ListenableFuture saveTimeseries(EntityId entityId, String key, Long value) { + protected ListenableFuture saveTimeseries(EntityId entityId, String key, Long value) { KvEntry telemetryValue = new LongDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); return timeseriesService.save(tenantId, entityId, timeseries); } + protected ListenableFuture saveTimeseries(EntityId entityId, String key, Boolean value) { + KvEntry telemetryValue = new BooleanDataEntry(key, value); + BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); + return timeseriesService.save(tenantId, entityId, timeseries); + } + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { diff --git a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java index 54e1a39769..8618e515c8 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java @@ -217,6 +217,90 @@ class TbelInvokeServiceTest extends AbstractTbelInvokeTest { assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull(); } + @Test + void givenForbiddenSocketHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.logging.SocketHandler(\"127.0.0.1\", 9999)"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.SocketHandler"); + }); + } + + @Test + void givenForbiddenZipFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.zip.ZipFile(\"/tmp/test.zip\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.zip.ZipFile"); + }); + } + + @Test + void givenForbiddenFileHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.logging.FileHandler(\"/tmp/test.log\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.FileHandler"); + }); + } + + @Test + void givenForbiddenJarFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.jar.JarFile(\"/tmp/test.jar\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.jar.JarFile"); + }); + } + + @Test + void givenForbiddenPreferences_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("java.util.prefs.Preferences.userRoot()"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getMessage()).contains("unresolvable property or identifier: java"); + }); + } + + @Test + void givenForbiddenLocaleServiceProvider_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.spi.LocaleServiceProvider()"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.spi.LocaleServiceProvider"); + }); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScriptResultString(scriptId, "{}"); diff --git a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java index 51e35bd999..f2485dc7d5 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java @@ -31,9 +31,12 @@ import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.service.install.DatabaseSchemaSettingsService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.system.SystemPatchApplier; @@ -41,6 +44,7 @@ import org.thingsboard.server.service.system.SystemPatchApplier; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -81,6 +85,9 @@ public class SystemPatchApplierTest { @Mock private WidgetTypeService widgetTypeService; + @Mock + private WidgetsBundleService widgetsBundleService; + @Mock private ImageService imageService; @@ -155,19 +162,72 @@ public class SystemPatchApplierTest { } @Test - void whenWidgetNotFound_thenThrowException() throws Exception { + void whenWidgetNotFound_thenCreateNewWidget() throws Exception { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); - WidgetTypeDetails testWidget = createTestWidgetType("test_widget", "Test Widget"); - String json = JacksonUtil.toString(testWidget); + WidgetTypeDetails fileWidget = createTestWidgetType("new_widget", "New Widget"); + String json = JacksonUtil.toString(fileWidget); assertNotNull(json); - Files.writeString(widgetTypesDir.resolve("test_widget.json"), json); + Files.writeString(widgetTypesDir.resolve("new_widget.json"), json); + + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "new_widget")).thenReturn(null); + + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + + assertNotNull(stats); + assertEquals(1, stats.created()); + assertEquals(0, stats.updated()); + verify(widgetTypeService).saveWidgetType(argThat(w -> "new_widget".equals(w.getFqn()))); + } + + @Test + void whenFqnIsBlank_thenThrowException() throws Exception { + Path widgetTypesDir = tempDir.resolve("widget_types"); + Files.createDirectories(widgetTypesDir); + when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); - when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")).thenReturn(null); + WidgetTypeDetails brokenWidget = createTestWidgetType("", "Broken Widget"); + String json = JacksonUtil.toString(brokenWidget); + assertNotNull(json); + Files.writeString(widgetTypesDir.resolve("broken.json"), json); assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes")); + verify(widgetTypeService, never()).saveWidgetType(any()); + } + + @Test + void whenMixOfCreatedAndUpdated_thenStatsAreCorrect() throws Exception { + Path widgetTypesDir = tempDir.resolve("widget_types"); + Files.createDirectories(widgetTypesDir); + when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + + WidgetTypeDetails newFileWidget = createTestWidgetType("widget_new", "Widget New"); + Files.writeString(widgetTypesDir.resolve("widget_new.json"), JacksonUtil.toString(newFileWidget)); + + WidgetTypeDetails changedFileWidget = createTestWidgetType("widget_changed", "Widget Changed New Name"); + Files.writeString(widgetTypesDir.resolve("widget_changed.json"), JacksonUtil.toString(changedFileWidget)); + + WidgetTypeDetails sameFileWidget = createTestWidgetType("widget_same", "Widget Same"); + Files.writeString(widgetTypesDir.resolve("widget_same.json"), JacksonUtil.toString(sameFileWidget)); + + WidgetTypeDetails existingChanged = createTestWidgetType("widget_changed", "Widget Changed Old Name"); + existingChanged.setId(new WidgetTypeId(UUID.randomUUID())); + + WidgetTypeDetails existingSame = createTestWidgetType("widget_same", "Widget Same"); + existingSame.setId(new WidgetTypeId(UUID.randomUUID())); + + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_new")).thenReturn(null); + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_changed")).thenReturn(existingChanged); + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_same")).thenReturn(existingSame); + + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + + assertNotNull(stats); + assertEquals(1, stats.created()); + assertEquals(1, stats.updated()); + verify(widgetTypeService, times(2)).saveWidgetType(any()); } @Test @@ -189,9 +249,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(1, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(1, stats.updated()); verify(widgetTypeService).saveWidgetType(argThat(w -> w.getDescriptor().get("version").asInt() == 2 )); @@ -214,9 +276,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(1, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(1, stats.updated()); verify(widgetTypeService).saveWidgetType(argThat(w -> "New Name".equals(w.getName()))); } @@ -237,9 +301,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(0, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(0, stats.updated()); verify(widgetTypeService, never()).saveWidgetType(any()); } @@ -339,8 +405,8 @@ public class SystemPatchApplierTest { // Simulate work while holding lock Thread.sleep(100); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - firstThreadSavedWidget.set(updated != null && updated > 0); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + firstThreadSavedWidget.set(stats != null && stats.updated() > 0); ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock"); } @@ -360,8 +426,8 @@ public class SystemPatchApplierTest { secondThreadAcquiredLock.set(Boolean.TRUE.equals(acquired)); if (secondThreadAcquiredLock.get()) { - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - secondThreadSavedWidget.set(updated != null && updated > 0); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + secondThreadSavedWidget.set(stats != null && stats.updated() > 0); ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock"); } @@ -560,6 +626,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded"); @@ -607,6 +674,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded"); @@ -834,6 +902,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); when(imageService.getAllImageKeysByTenantId(TenantId.SYS_TENANT_ID)).thenReturn(Collections.emptySet()); @@ -854,4 +923,201 @@ public class SystemPatchApplierTest { verify(imageService, never()).createOrUpdateSystemImage(anyString(), any(byte[].class)); } + // --- updateWidgetBundles tests --- + + @Test + void whenWidgetBundlesDirDoesNotExist_thenReturnsZero() { + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("missing_bundles")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleNotInDb_thenSkipWithoutCreation() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\"]}"); + + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(null); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleExistsAndHasNewFqn_thenMergeFqns() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\",\"new_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart", "bar_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(1, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService).updateWidgetsBundleWidgetFqns( + eq(TenantId.SYS_TENANT_ID), + eq(existingBundle.getId()), + argThat(fqns -> fqns.size() == 3 + && fqns.get(0).equals("line_chart") + && fqns.get(1).equals("bar_chart") + && fqns.get(2).equals("new_chart")) + ); + } + + @Test + void whenBundleExistsAndAllFqnsAlreadyLinked_thenNoLinkUpdate() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart", "bar_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenOnlyBundleImageFormatDiffers_thenNoUpdate() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + // File carries a base64 data URI; DB has the resolved system-image URL — same content, different format. + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10," + + "\"image\":\"data:image/png;base64,iVBORw0KGgo\"}," + + "\"widgetTypeFqns\":[]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + existingBundle.setImage("tb-image;/api/images/system/charts.png"); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of()); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleMetadataChanged_thenUpdateBundle() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"New Title\",\"description\":\"new\",\"order\":20}," + + "\"widgetTypeFqns\":[\"line_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Old Title"); + existingBundle.setDescription("old"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(1, updated); + verify(widgetsBundleService).saveWidgetsBundle(argThat(b -> + "New Title".equals(b.getTitle()) && "new".equals(b.getDescription()) && b.getOrder() == 20 + )); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleAliasIsBlank_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("broken.json"), + "{\"widgetsBundle\":{\"alias\":\"\",\"title\":\"Broken\"}}"); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + @Test + void whenBundleJsonMissingWidgetsBundleField_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("broken.json"), "{\"foo\":\"bar\"}"); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + @Test + void whenBundleHasInlineWidgetTypes_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypes\":[" + + "{\"fqn\":\"inline_chart\",\"name\":\"Inline\",\"descriptor\":{\"type\":\"latest\"}}" + + "]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetTypeService, never()).saveWidgetType(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + private WidgetsBundle createTestBundle(String alias, String title) { + WidgetsBundle bundle = new WidgetsBundle(); + bundle.setId(new WidgetsBundleId(UUID.randomUUID())); + bundle.setAlias(alias); + bundle.setTitle(title); + bundle.setTenantId(TenantId.SYS_TENANT_ID); + return bundle; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java b/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java index e1f0b84a4d..9e91adb2c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java @@ -25,12 +25,10 @@ import java.util.List; public record TbUserMessage( @NotEmpty @Valid - @ArraySchema( - arraySchema = @Schema( - requiredMode = Schema.RequiredMode.REQUIRED, - description = "A list of content parts that make up the complete user prompt" - ), - schema = @Schema(ref = "#/components/schemas/TbContent") + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "A list of content parts that make up the complete user prompt" ) + @ArraySchema(schema = @Schema(ref = "#/components/schemas/TbContent")) List contents ) {} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java index a20d0dca55..532f7f2e1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java @@ -37,17 +37,21 @@ public record AvailableEntityKeys( ) Set entityTypes, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "List of unique time series key names available on the matched entities." + ) @ArraySchema( - arraySchema = @Schema(description = "List of unique time series key names available on the matched entities."), schema = @Schema(implementation = String.class, example = "temperature"), uniqueItems = true ) List timeseries, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "List of unique attribute key names available on the matched entities." + ) @ArraySchema( - arraySchema = @Schema(description = "List of unique attribute key names available on the matched entities."), schema = @Schema(implementation = String.class, example = "serialNumber"), uniqueItems = true ) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java index 536b70e4ff..608177b9ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java @@ -41,15 +41,13 @@ public record AvailableEntityKeysV2( ) Set entityTypes, - @ArraySchema( - arraySchema = @Schema( - description = """ - List of unique time series keys available on the matched entities, sorted alphabetically. - Omitted when timeseries keys were not requested.""", - nullable = true - ), - schema = @Schema(implementation = KeyInfo.class) + @Schema( + description = """ + List of unique time series keys available on the matched entities, sorted alphabetically. + Omitted when timeseries keys were not requested.""", + nullable = true ) + @ArraySchema(schema = @Schema(implementation = KeyInfo.class)) @Nullable List timeseries, @Schema( diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java index 62140abdd1..6883fff6d4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java @@ -36,7 +36,8 @@ public class WidgetsBundleExportData extends EntityExportData { @Override public EntityType getEntityType() { return EntityType.WIDGETS_BUNDLE; } - @ArraySchema(arraySchema = @Schema(description = "List of widgets in the bundle"), schema = @Schema(implementation = JsonNode.class)) + @Schema(description = "List of widgets in the bundle") + @ArraySchema(schema = @Schema(implementation = JsonNode.class)) @JsonProperty(index = 3) private List widgets; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 87f566c90f..dcf9459965 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -98,7 +98,10 @@ public abstract class AbstractTbQueueConsumerTemplate i doSubscribe(partitions); subscribed = true; } - records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); + if (partitions.isEmpty()) { + return sleepAndReturnEmpty(startNanos, durationInMillis); + } + records = doPoll(durationInMillis); } finally { consumerLock.unlock(); } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java new file mode 100644 index 0000000000..a016408ad4 --- /dev/null +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.BDDMockito.never; +import static org.mockito.BDDMockito.spy; +import static org.mockito.BDDMockito.times; +import static org.mockito.BDDMockito.verify; + +@Slf4j +@ExtendWith(MockitoExtension.class) +public class AbstractTbQueueConsumerTemplateTest { + + private static final long POLL_DURATION_MS = 100L; + private static final long SLEEP_TOLERANCE_MS = 20L; + + @Test + public void givenEmptyPartitionsAndLongPollingSupported_whenPoll_thenSleepsAndDoesNotCallDoPoll() { + // Regression: with empty partitions AND isLongPollingSupported()==true (e.g. Kafka), + // poll() previously returned instantly with no sleep, causing the consumer loop to busy-spin. + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, never()).doPoll(anyLong()); + assertThat("poll() must sleep ~durationInMillis when partitions are empty (no busy-wait)", + elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + @Test + public void givenEmptyPartitionsAndNoLongPolling_whenPoll_thenSleepsAndDoesNotCallDoPoll() { + TestConsumer consumer = spy(new TestConsumer("test-topic", false)); + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, never()).doPoll(anyLong()); + assertThat(elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + @Test + public void givenNonEmptyPartitions_whenPoll_thenCallsDoPoll() { + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.singleton(new TopicPartitionInfo("test-topic", null, 0, true))); + + List result = consumer.poll(POLL_DURATION_MS); + + assertThat(result, is(empty())); + verify(consumer, times(1)).doPoll(POLL_DURATION_MS); + } + + @Test + public void givenPartitionsBecomeEmptyAfterRebalance_whenPollAgain_thenStopsCallingDoPoll() { + // Reproduces the observed trigger: a rebalance leaves the consumer with an empty + // partition assignment. Subsequent poll() calls must not busy-spin or call doPoll(). + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.singleton(new TopicPartitionInfo("test-topic", null, 0, true))); + consumer.poll(POLL_DURATION_MS); + verify(consumer, times(1)).doPoll(POLL_DURATION_MS); + + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, times(1)).doPoll(anyLong()); + assertThat(elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + static class TestConsumer extends AbstractTbQueueConsumerTemplate { + + private final boolean longPollingSupported; + + TestConsumer(String topic, boolean longPollingSupported) { + super(topic); + this.longPollingSupported = longPollingSupported; + } + + @Override + protected List doPoll(long durationInMillis) { + return Collections.emptyList(); + } + + @Override + protected TbQueueMsg decode(Object record) { + return null; + } + + @Override + protected void doSubscribe(Set partitions) { + } + + @Override + protected void doCommit() { + } + + @Override + protected void doUnsubscribe() { + } + + @Override + protected boolean isLongPollingSupported() { + return longPollingSupported; + } + } + +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 7d92639f1a..af5e1f04b2 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -160,7 +160,6 @@ public class SnmpTransportContext extends TransportContext { return; } sessions.put(device.getId(), sessionContext); - snmpTransportService.createQueryingTasks(sessionContext); log.info("Established SNMP device session for device {}", device.getId()); } @@ -224,6 +223,8 @@ public class SnmpTransportContext extends TransportContext { registerTransportSession(sessionContext, msg); }); transportService.lifecycleEvent(sessionContext.getTenantId(), sessionContext.getDeviceId(), ComponentLifecycleEvent.STARTED, true, null); + snmpTransportService.createQueryingTasks(sessionContext); + log.info("[{}] Session registered and querying tasks created", sessionContext.getDeviceId()); } else { log.warn("[{}] Failed to process device auth", sessionContext.getDeviceId()); } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 0c39b7e740..a15a21b6e3 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -164,7 +164,7 @@ public class SnmpTransportService implements TbTransportService, CommandResponde ScheduledTask scheduledTask = new ScheduledTask(); scheduledTask.init(() -> { try { - if (sessionContext.isActive()) { + if (sessionContext.isActive() && sessionContext.isConnected()) { return sendRequest(sessionContext, repeatingCommunicationConfig); } } catch (Exception e) { @@ -390,7 +390,11 @@ public class SnmpTransportService implements TbTransportService, CommandResponde JsonObject responseData = responseDataMappers.get(requestContext.getCommunicationSpec()).map(response, requestContext); if (responseData.size() == 0) { - log.warn("[{}] No values in the response", sessionContext.getDeviceId()); + log.warn("[{}] No values in the response for spec {}. Response PDUs count: {}, Mappings count: {}", + sessionContext.getDeviceId(), requestContext.getCommunicationSpec(), + response.size(), requestContext.getResponseMappings().size()); + log.debug("[{}] No values in the response for spec {}. Response PDUs: {}", + sessionContext.getDeviceId(), requestContext.getCommunicationSpec(), response); throw new IllegalArgumentException("No values in the response"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index a9b04618c8..59964d1949 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.query; +import jakarta.annotation.PostConstruct; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -60,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -318,10 +320,19 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .replace("$in", "from").replace("$out", "to") .replace("$rootIdCondition", "in (:relation_root_ids)"); + private static final String NULLS_ORDER_DEFAULT = "default"; + private static final String NULLS_ORDER_FIRST = "nulls_first"; + private static final String NULLS_ORDER_LAST = "nulls_last"; + private static final Set ACCEPTED_NULLS_ORDER_STRATEGIES = Set.of(NULLS_ORDER_DEFAULT, NULLS_ORDER_FIRST, NULLS_ORDER_LAST); + @Getter @Value("${sql.relations.max_level:50}") int maxLevelAllowed; //This value has to be reasonable small to prevent infinite recursion as early as possible + @Getter + @Value("${sql.entity_data_query_nulls_order_strategy:default}") + String nullsOrderStrategy; + private final NamedParameterJdbcTemplate jdbcTemplate; private final TransactionTemplate transactionTemplate; private final DefaultQueryLogComponent queryLog; @@ -332,6 +343,15 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { this.queryLog = queryLog; } + @PostConstruct + void validateNullsOrderStrategy() { + if (!ACCEPTED_NULLS_ORDER_STRATEGIES.contains(nullsOrderStrategy)) { + log.error("Invalid value '{}' for sql.entity_data_query_nulls_order_strategy. Accepted values are: {}. Falling back to '{}'.", + nullsOrderStrategy, ACCEPTED_NULLS_ORDER_STRATEGIES, NULLS_ORDER_DEFAULT); + nullsOrderStrategy = NULLS_ORDER_DEFAULT; + } + } + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); @@ -502,11 +522,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { if (sortOrderMappingOpt.isPresent()) { EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get(); String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc"; + String nullsOrder = resolveNullsOrder(); if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) { - dataQuery = String.format("%s order by %s %s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, direction); + dataQuery = String.format("%s order by %s %s%s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, nullsOrder, direction); } else { - dataQuery = String.format("%s order by %s %s, %s %s, result.id %s", dataQuery, - sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction, direction); + dataQuery = String.format("%s order by %s %s%s, %s %s, result.id %s", dataQuery, + sortOrderMapping.getSortOrderNumAlias(), direction, nullsOrder, sortOrderMapping.getSortOrderStrAlias(), direction, direction); } } } @@ -525,6 +546,14 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } + private String resolveNullsOrder() { + return switch (nullsOrderStrategy) { + case NULLS_ORDER_FIRST -> " NULLS FIRST"; + case NULLS_ORDER_LAST -> " NULLS LAST"; + default -> ""; + }; + } + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index b9a134e713..51c926263f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -511,8 +511,8 @@ public class EntityKeyMapping { String attrNumAlias = getSortOrderNumAlias(); String attrVarcharAlias = getSortOrderStrAlias(); String attrSortOrderSelection = - String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," + - "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias); + String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v is null then null when %s.bool_v then 1 else 0 end)) %s," + + "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias); return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection); } else { return String.join(", ", attrValSelection, attrTsSelection); diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 010994dc88..4dc404e2ca 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -103,8 +103,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 91c4927b3c..df29e26e5e 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -358,6 +358,16 @@ public class TestRestClient { .statusCode(HTTP_OK); } + public JsonNode testRuleChainScript(Object body) { + return given().spec(requestSpec) + .body(body) + .post("/api/ruleChain/testScript") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + private String getUrlParams(PageLink pageLink) { String urlParams = "pageSize={pageSize}&page={page}"; if (!isEmpty(pageLink.getTextSearch())) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java new file mode 100644 index 0000000000..1328382c5d --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.security; + +import com.fasterxml.jackson.databind.JsonNode; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.thingsboard.server.msa.AbstractContainerTest; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JsExecutorSandboxIsolationTest extends AbstractContainerTest { + + @BeforeClass + public void beforeClass() { + testRestClient.login("tenant@thingsboard.org", "tenant"); + } + + @AfterClass + public void afterClass() { + testRestClient.resetToken(); + } + + /** + * Black-box regression for JVN#16937365: a tenant admin must not be able + * to escape the tb-js-executor sandbox via the host-realm prototype chain + * exposed through the script's `args` argument. Runs against the live + * docker-compose deployment, which uses script.use_sandbox=true and + * JS_EVALUATOR=remote (Kafka -> tb-js-executor). + */ + @Test + public void testRuleChainScriptCannotReachHostProcess() { + JsonNode response = testRestClient.testRuleChainScript(""" + { + "script": "var F = args.constructor.constructor; var p = F('return process')(); return { reachedHost: !!(p && p.mainModule) };", + "scriptType": "update", + "argNames": ["msg", "metadata", "msgType"], + "msg": "{}", + "metadata": {}, + "msgType": "POST_TELEMETRY_REQUEST" + } + """); + + // The sandboxed run must reject the escape attempt: the host `process` + // global is not defined inside the sandbox realm, so executing the + // synthesized function `F("return process")` throws. + assertThat(response.has("error")).isTrue(); + String error = response.get("error").asText(); + assertThat(error) + .as("sandbox must block host-realm reach via args.constructor.constructor; full error: %s", error) + .contains("process is not defined"); + + // Defense in depth: even if the script somehow returned, output must + // not indicate that the host process was reached. + if (response.hasNonNull("output")) { + assertThat(response.get("output").asText()).doesNotContain("\"reachedHost\":true"); + } + } +} diff --git a/msa/black-box-tests/src/test/resources/connectivity.xml b/msa/black-box-tests/src/test/resources/connectivity.xml index a1cbaf4af6..7cd96bb429 100644 --- a/msa/black-box-tests/src/test/resources/connectivity.xml +++ b/msa/black-box-tests/src/test/resources/connectivity.xml @@ -25,6 +25,7 @@ + \ No newline at end of file diff --git a/msa/js-executor/api/jsExecutor.ts b/msa/js-executor/api/jsExecutor.ts index 855ad99932..ab916a2760 100644 --- a/msa/js-executor/api/jsExecutor.ts +++ b/msa/js-executor/api/jsExecutor.ts @@ -15,14 +15,22 @@ /// import vm, { Script } from 'vm'; +import { _logger } from '../config/logger'; export type TbScript = Script | Function; export class JsExecutor { useSandbox: boolean; + private logger = _logger('JsExecutor'); constructor(useSandbox: boolean) { this.useSandbox = useSandbox; + if (!useSandbox) { + this.logger.warn( + 'script.use_sandbox=false: dangerous by design — user-supplied scripts run in the host realm with no isolation. ' + + 'Use only as a performance trade-off in trusted, non-public clusters.' + ); + } } compileScript(code: string): Promise { @@ -56,9 +64,15 @@ export class JsExecutor { private invokeScript(script: Script, args: string[], timeout: number | undefined): Promise { return new Promise((resolve, reject) => { try { - const sandbox = Object.create(null); - sandbox.args = args; - const result = script.runInNewContext(sandbox, {timeout: timeout}); + const sandbox = vm.createContext(Object.create(null)); + // Construct args inside the sandbox context so it inherits sandbox-realm + // prototypes; prevents prototype-based escapes from the host realm. + const ctxArgs = vm.runInContext('[]', sandbox) as string[]; + for (let i = 0; i < args.length; i++) { + ctxArgs[i] = String(args[i]); + } + sandbox.args = ctxArgs; + const result = script.runInContext(sandbox, {timeout: timeout}); resolve(result); } catch (err) { reject(err); @@ -67,6 +81,11 @@ export class JsExecutor { } + // DANGEROUS BY DESIGN: the non-sandbox path. vm.compileFunction's + // parsingContext only isolates *parsing*, not *execution* — the resulting + // function runs in the host realm with full access to host globals + // (process, require, etc.). Enabled only via script.use_sandbox=false as + // a performance trade-off in trusted clusters. private createFunction(code: string): Promise { return new Promise((resolve, reject) => { try { diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 6a98361c72..9923fe4ed3 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -34,7 +34,7 @@ kafka: partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" connectionTimeout: "TB_KAFKA_CONNECTION_TIMEOUT_MS" - compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed + compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip, lz4 or none topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" use_confluent_cloud: "TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD" client_id: "KAFKA_CLIENT_ID" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index be49ece5a6..9a33190069 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -34,7 +34,7 @@ kafka: partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 connectionTimeout: "1000" # The default value in kafkajs is: 1000 - compression: "none" # gzip or uncompressed + compression: "none" # gzip, lz4 or none topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" use_confluent_cloud: false client_id: "kafkajs" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh @@ -50,6 +50,12 @@ logger: filename: "tb-js-executor-%DATE%.log" script: + # WARNING: setting this to "false" is DANGEROUS BY DESIGN. The non-sandbox + # path compiles and runs user-supplied scripts in the host realm via + # vm.compileFunction; it provides no isolation and exposes the host process + # (file system, environment variables, child_process, etc.) to script + # authors. Use "false" only as a performance trade-off in trusted, + # non-public clusters where every script author is fully trusted. use_sandbox: "true" memory_usage_trace_frequency: "1000" script_body_trace_frequency: "10000" diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 61ecce6fc3..5e0c41fe85 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -6,13 +6,14 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --out-path ./target ./target/src && node install.js", - "test": "echo \"Error: no test specified\" && exit 1", + "pkg": "tsc && pkg -t node22-linux-x64 --output ./target/thingsboard-js-executor-linux ./target/src && pkg -t node22-win-x64 --no-bytecode --public-packages \"*\" --public --output ./target/thingsboard-js-executor-win.exe ./target/src && node install.js", + "test": "mkdir -p target/surefire-reports && node --require ts-node/register --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=target/surefire-reports/TEST-js-executor.xml test/jsExecutor.test.ts", "start": "nodemon --watch '.' --ext 'ts' --exec 'ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'NODE_ENV=production ts-node server.ts'", "build": "tsc" }, "dependencies": { + "@2l/kafkajs-lz4": "^1.3.2", "config": "^4.1.1", "express": "^5.1.0", "js-yaml": "^4.1.1", @@ -46,7 +47,8 @@ }, "pkg": { "assets": [ - "node_modules/config/**/*.*" + "node_modules/config/**/*.*", + "node_modules/@antoniomuso/lz4-napi-*/**/*.node" ] } } diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index c47b40d561..4396f4fb96 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -116,6 +116,17 @@ --mutex network run pkg + + yarn test + + yarn + + test + + ${maven.test.skip} + --mutex network run test + + diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 4527e9e1ce..175abd80f4 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -21,6 +21,7 @@ import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' import { IQueue } from './queue.models'; import { Admin, + CompressionCodecs, CompressionTypes, Consumer, Kafka, @@ -46,7 +47,31 @@ export class KafkaTemplate implements IQueue { private linger = Number(config.get('kafka.linger_ms')); private requestTimeout = Number(config.get('kafka.requestTimeout')); private connectionTimeout = Number(config.get('kafka.connectionTimeout')); - private compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; + private compressionType = this.resolveCompressionType(config.get('kafka.compression')); + + private resolveCompressionType(compression: string): CompressionTypes { + switch (compression) { + case 'gzip': + return CompressionTypes.GZIP; + case 'lz4': { + // Load the LZ4 codec lazily so users who don't enable LZ4 don't take a hard + // dependency on the lz4-napi native binary (e.g. inside pkg-built executables). + // The package re-assigns module.exports = LZ4Codec, which wipes the __esModule + // flag and the .default property — so accept either shape. + const lz4Module = require('@2l/kafkajs-lz4'); + const LZ4Codec = lz4Module.default || lz4Module; + CompressionCodecs[CompressionTypes.LZ4] = new LZ4Codec().codec; + return CompressionTypes.LZ4; + } + case 'none': + return CompressionTypes.None; + default: + if (isNotEmptyStr(compression)) { + this.logger.warn('Unknown kafka.compression value "%s"; falling back to no compression. Supported values: gzip, lz4, none.', compression); + } + return CompressionTypes.None; + } + } private partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently')); private kafkaClient: Kafka; diff --git a/msa/js-executor/test/jsExecutor.test.ts b/msa/js-executor/test/jsExecutor.test.ts new file mode 100644 index 0000000000..7777030ce9 --- /dev/null +++ b/msa/js-executor/test/jsExecutor.test.ts @@ -0,0 +1,83 @@ +/// +/// Copyright © 2016-2026 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { JsExecutor } from '../api/jsExecutor'; + +// describe('js-executor') groups all cases under +// in the JUnit XML so they show up under that suite in TeamCity's Tests tab, +// alongside thousands of Java tests. +describe('js-executor', () => { + +test('sandbox isolates args from host realm (JVN#16937365)', async () => { + const exec = new JsExecutor(true); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + var F = args.constructor.constructor; + var p = F("return process")(); + return p && p.mainModule ? 'reached-host' : 'isolated'; + }`); + await assert.rejects( + exec.executeScript(script, ['{}', '{}', 'POST_TELEMETRY_REQUEST'], 5000), + /process is not defined/, + 'host process must not be reachable from inside the sandbox', + ); +}); + +test('sandbox passes string args through unchanged', async () => { + const exec = new JsExecutor(true); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + return { msgIsString: typeof msg === 'string', count: args.length, first: args[0] }; + }`); + const out = await exec.executeScript(script, ['hello', '{}', 'X'], 5000); + // Field-by-field: the returned object is owned by the sandbox realm, so + // its prototype is not the host Object.prototype and deepStrictEqual would + // reject it on prototype mismatch even when the values match. + assert.equal(out.msgIsString, true); + assert.equal(out.count, 3); + assert.equal(out.first, 'hello'); +}); + +// The use_sandbox=false path is intentionally non-isolating: scripts compile +// and run in the host realm via vm.compileFunction. The two tests below codify +// that documented contract so any future behavior change shows up as a test +// failure and forces a deliberate update of the docs and threat model. + +test('non-sandbox path does not isolate from host realm (documented contract)', async () => { + const exec = new JsExecutor(false); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + // Non-destructive host-reach probe: typeof process.platform is 'string' + // only if the host process object is reachable. + var F = args.constructor.constructor; + return F('return typeof process.platform')(); + }`); + const out = await exec.executeScript(script, ['{}', '{}', 'X']); + assert.equal(out, 'string', + 'use_sandbox=false is documented as non-isolating; if this fails, the path was changed and docs/threat model must be updated'); +}); + +test('non-sandbox path passes string args through unchanged', async () => { + const exec = new JsExecutor(false); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + return { msgIsString: typeof msg === 'string', count: args.length, first: args[0] }; + }`); + const out = await exec.executeScript(script, ['hello', '{}', 'X']); + assert.equal(out.msgIsString, true); + assert.equal(out.count, 3); + assert.equal(out.first, 'hello'); +}); + +}); // describe('js-executor') diff --git a/msa/js-executor/tsconfig.json b/msa/js-executor/tsconfig.json index b633ffc768..a1f7b25466 100644 --- a/msa/js-executor/tsconfig.json +++ b/msa/js-executor/tsconfig.json @@ -9,5 +9,5 @@ "skipLibCheck": true, "strictPropertyInitialization": false }, - "exclude": ["node_modules", "target"] + "exclude": ["node_modules", "target", "test"] } diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 7e2fe58a13..5e3330e0d7 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -2,6 +2,78 @@ # yarn lockfile v1 +"@2l/kafkajs-lz4@^1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@2l/kafkajs-lz4/-/kafkajs-lz4-1.3.2.tgz#25259f693a709816ef4eb62514c1ce1bab80502f" + integrity sha512-yq5dx4CbL2sofWXKuadyty3ZKRcZyqe5iiuuOWjZgtIed9MBZ2Wqe5hQN18+a5XVfR4SFQGVISb/a0oTjdPjwQ== + dependencies: + lz4-napi "^2.8.0" + +"@antoniomuso/lz4-napi-android-arm-eabi@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-android-arm-eabi/-/lz4-napi-android-arm-eabi-2.9.0.tgz#5ea67847f0a761c9aec22a31212d0e429ca01fb2" + integrity sha512-aeT/9SoWq7rnmzssWuCKUPaxVt3fzE9q+xq/ZHbnUSmrm8/EhLOACMvQeCOnL0IZsmPh8EpuwIE1TZyM9iQPRA== + +"@antoniomuso/lz4-napi-android-arm64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-android-arm64/-/lz4-napi-android-arm64-2.9.0.tgz#64cb0aac70267bb071bba2d8e382301e058811ca" + integrity sha512-ibQ0qiEvmljXAM97IgOZfh+PeiSQ0Rqf2HErJlZPVm2v4GVJoB67v21v1TUydqNNV5L8bwufVoZ90nheL8X9ZA== + +"@antoniomuso/lz4-napi-darwin-arm64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-darwin-arm64/-/lz4-napi-darwin-arm64-2.9.0.tgz#669e48b165af11cec20581acc2caa0d1c0f84472" + integrity sha512-1su4K1MWa4bcWoZlHajv+luGmFDV1JwIsvjtDF+0HhUveSDPP+8A4Z34zOZidURIr08Sl7M7ViPth6ZQ9SqnAA== + +"@antoniomuso/lz4-napi-darwin-x64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-darwin-x64/-/lz4-napi-darwin-x64-2.9.0.tgz#a00ddf021772e26bd01b3ca5e3025f11d1667edf" + integrity sha512-8Lnbm2MkdJtiJ/nbcRS9zRyGp3G0sG6D+Y/x1vTP8nZs3/f8tBwYNsjxCQyyXNNyHcYWwVGbk68onP/pyDljOA== + +"@antoniomuso/lz4-napi-freebsd-x64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-freebsd-x64/-/lz4-napi-freebsd-x64-2.9.0.tgz#fdc2b292489bd6d4e44eb7bb059fb9bd4b860f14" + integrity sha512-k04EMVOjntKDPrdR4Tf8WyNseuk9PTtSGw8WHyp4CTjoR1s+YJxtp9SMnThe5o2q0TATwk8WGYb/Howrp5OMxw== + +"@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm-gnueabihf/-/lz4-napi-linux-arm-gnueabihf-2.9.0.tgz#9fcdf93b3ca5aa24469e4bf11dd7100df8c71975" + integrity sha512-H92F8zPZmgy2r8IhCWh3qIBfLp2BQ5cp18RoDXhtGFWwkh+5gVWrZp11IVznrsdgB0QeW0VR7dAMMHg3WLOPfA== + +"@antoniomuso/lz4-napi-linux-arm64-gnu@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm64-gnu/-/lz4-napi-linux-arm64-gnu-2.9.0.tgz#9d22e075a9cb60c3cb415428138618ae4a79f3c5" + integrity sha512-25crh0qs/3Rj3fMI8ulYD0DoaKsidUhMBki2aeO69ZK+F8bmQ/e2++FlgJ6f3EgMP5CNxJtnZXKhPOraQWjwAw== + +"@antoniomuso/lz4-napi-linux-arm64-musl@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm64-musl/-/lz4-napi-linux-arm64-musl-2.9.0.tgz#d957bce3accd49199bb6de522f5163efa2068fe4" + integrity sha512-eJtHp38zuLaYI0/cOV/BKcNQiXUBo4GPx53FTf0Y307yUjLsn48LNeN0vD28Ct9YrbUae3bQvMD5AD86She0ww== + +"@antoniomuso/lz4-napi-linux-x64-gnu@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-x64-gnu/-/lz4-napi-linux-x64-gnu-2.9.0.tgz#c86567d5aa23863059afbee829fb9f45895d3869" + integrity sha512-mDjS4dyjRKaZQcAP71SphkYH5r3kufB30ih/VETVu/br2toCfBk6Zr1xhL1r+L7FaVAFzF62B7h30CiqrN0Awg== + +"@antoniomuso/lz4-napi-linux-x64-musl@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-x64-musl/-/lz4-napi-linux-x64-musl-2.9.0.tgz#7d02612dec7d6247645aa321871b15247da0ce7d" + integrity sha512-pvU7Z7qjkjn17NkddBtBQ7C2iRqjtZ7WJ3Jqrjtj4XxolY3Q0HaYMvWjkWhzb9AKGZbj5y+EHYtbVoZJ2TSQhQ== + +"@antoniomuso/lz4-napi-win32-arm64-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-arm64-msvc/-/lz4-napi-win32-arm64-msvc-2.9.0.tgz#c901bfec718303ed86867129ecc0371eaa883517" + integrity sha512-aioLlbpJl0QPEXLXhh2bzyitc3T7Jot3f1ap6WdKiRa+CIjMHXw1nxJXy07MLXif10r+qVZr86ic8dvwErgqEQ== + +"@antoniomuso/lz4-napi-win32-ia32-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-ia32-msvc/-/lz4-napi-win32-ia32-msvc-2.9.0.tgz#11e48b18b7923c250735e0130998e5968ae91130" + integrity sha512-VaF4XMTdYb59TsPsiqnWwsNaWKHhgxF33z5p4zg4n0tp20eWozl76hn8B+aXthSs40W0W1N97QhxxV4oXGd8cg== + +"@antoniomuso/lz4-napi-win32-x64-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-x64-msvc/-/lz4-napi-win32-x64-msvc-2.9.0.tgz#20d9f71a638f3277cd0b7662e966f90e53d98af8" + integrity sha512-wfA8ShO3eGLxJ1LDwXJo87XL2D4NkMJV1pfHPvLZpD0MWb9u8VfgS+gKK5YhT7XKjzVdeIna9jgFdn2HBnZBxA== + "@babel/generator@^7.23.0": version "7.28.3" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.28.3.tgz#9626c1741c650cbac39121694a0f2d7451b8ef3e" @@ -100,6 +172,18 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@napi-rs/triples@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@napi-rs/triples/-/triples-1.2.0.tgz#bcd9c936acb93890e7015818e0181f3db421aafa" + integrity sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA== + +"@node-rs/helper@^1.3.3": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@node-rs/helper/-/helper-1.6.0.tgz#83e5381de6e898d0b8c92178bb8d897d619e3a3a" + integrity sha512-2OTh/tokcLA1qom1zuCJm2gQzaZljCCbtX1YCrwRVd/toz7KxaDRFeLTAPwhs8m9hWgzrBn5rShRm6IaZofCPw== + dependencies: + "@napi-rs/triples" "^1.2.0" + "@tsconfig/node10@^1.0.7": version "1.0.11" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" @@ -998,6 +1082,27 @@ long@^5.3.2: resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== +lz4-napi@^2.8.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/lz4-napi/-/lz4-napi-2.9.0.tgz#4a700974a1154f82b3c0e1b7030fe63140051e7d" + integrity sha512-ZOWqxBMIK5768aD20tYn5B6Pp9WPM9UG/LHk8neG9p0gC1DtjdzhTtlkxhAjvTRpmJvMtnnqLKlT+COlqAt9cQ== + dependencies: + "@node-rs/helper" "^1.3.3" + optionalDependencies: + "@antoniomuso/lz4-napi-android-arm-eabi" "2.9.0" + "@antoniomuso/lz4-napi-android-arm64" "2.9.0" + "@antoniomuso/lz4-napi-darwin-arm64" "2.9.0" + "@antoniomuso/lz4-napi-darwin-x64" "2.9.0" + "@antoniomuso/lz4-napi-freebsd-x64" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm-gnueabihf" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm64-gnu" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm64-musl" "2.9.0" + "@antoniomuso/lz4-napi-linux-x64-gnu" "2.9.0" + "@antoniomuso/lz4-napi-linux-x64-musl" "2.9.0" + "@antoniomuso/lz4-napi-win32-arm64-msvc" "2.9.0" + "@antoniomuso/lz4-napi-win32-ia32-msvc" "2.9.0" + "@antoniomuso/lz4-napi-win32-x64-msvc" "2.9.0" + make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 56e15d4b4d..ab806701e5 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,7 +73,7 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 683a995084..ce18b0b063 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -6,7 +6,7 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --out-path ./target ./target/src && node install.js", + "pkg": "tsc && pkg -t node22-linux-x64 --output ./target/thingsboard-web-ui-linux ./target/src && pkg -t node22-win-x64 --no-bytecode --public-packages \"*\" --public --output ./target/thingsboard-web-ui-win.exe ./target/src && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web NODE_ENV=production ts-node server.ts'", diff --git a/pom.xml b/pom.xml index f2d3f280b1..306879f4fa 100755 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,8 @@ 3.5.13 3.18.0 42.7.11 + 4.1.133.Final + 10.1.55 2.4.0-b180830.0359 0.12.5 0.10 @@ -93,7 +95,7 @@ 3.9.5 3.25.5 1.76.0 - 1.2.9 + 1.2.10 1.18.46 1.2.5 1.2.5 @@ -142,6 +144,7 @@ 1.7.5 3.8.0 1.8.0-TB + 2.5.9 2.38.0 1.24 1.11.0 @@ -1006,6 +1009,32 @@ + + + io.netty + netty-bom + ${netty.version} + pom + import + + + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-websocket + ${tomcat.version} + + org.springframework.boot spring-boot-dependencies @@ -1354,6 +1383,11 @@ postgresql ${postgresql.version} + + org.apache.opennlp + opennlp-tools + ${opennlp-tools.version} + commons-io commons-io diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index db44521f30..f55f155b45 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -285,8 +285,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 2f8d730c9f..69e610a812 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -235,7 +235,7 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index ec31aec9e9..05fe2c2956 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -335,8 +335,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index aa0a6ac30c..1ff998560a 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -268,8 +268,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 71524ba47e..bb46478477 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -207,8 +207,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts index 060a74aebd..541b6ab384 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts @@ -24,7 +24,7 @@ import { ViewChild, ViewEncapsulation } from '@angular/core'; -import { DomSanitizer, SafeUrl } from '@angular/platform-browser'; + import { ImageService } from '@app/core/public-api'; import { AppState } from '@core/core.state'; import { AttributeService } from '@core/http/attribute.service'; @@ -63,8 +63,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On protected store: Store, private imageService: ImageService, private utils: UtilsService, - private attributeService: AttributeService, - private sanitizer: DomSanitizer + private attributeService: AttributeService ) { super(store); } @@ -115,8 +114,8 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On isLoading = false; singleDevice = true; updatePhoto = false; - previewPhoto: SafeUrl; - lastPhoto: SafeUrl; + previewPhoto: string; + lastPhoto: string; datasourceDetected = false; private mimeType: string; @@ -176,7 +175,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On private updateWidgetData(data: Array) { const keyData = data[0].data; if (keyData?.length && isString(keyData[0][1])) { - this.lastPhoto = keyData[0][1].startsWith('data:image/') ? this.sanitizer.bypassSecurityTrustUrl(keyData[0][1]) : keyData[0][1]; + this.lastPhoto = keyData[0][1]; } } @@ -309,7 +308,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On const file = new File([blob], fileName, { type: this.mimeType }); return this.imageService.uploadImage(file, fileName); }), - map((imageInfo) => + map((imageInfo) => this.settings.usePublicGalleryLink ? imageInfo.publicLink : imageInfo.link ) ); diff --git a/ui-ngx/src/app/shared/import-export/import-export.models.ts b/ui-ngx/src/app/shared/import-export/import-export.models.ts index 2476922bf9..ab9def1697 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.models.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.models.ts @@ -210,7 +210,7 @@ function splitCSV(str: string, sep: string): string[] { foo = foo.shift().split(sep).concat(foo); } } else { - foo[x].replace(/""/g, '"'); + foo[x] = foo[x].replace(/""/g, '"'); } } return foo; diff --git a/ui-ngx/src/app/shared/models/page/page-link.ts b/ui-ngx/src/app/shared/models/page/page-link.ts index 023821587b..eb2c059c2a 100644 --- a/ui-ngx/src/app/shared/models/page/page-link.ts +++ b/ui-ngx/src/app/shared/models/page/page-link.ts @@ -84,11 +84,25 @@ export function sortItems(item1: any, item2: any, property: string, asc: boolean result = item1Value - item2Value; } else if (item1Type === 'string' && item2Type === 'string') { result = item1Value.localeCompare(item2Value); - } else if ((item1Type === 'boolean' && item2Type === 'boolean') || (item1Type !== item2Type)) { - if (item1Value && !item2Value) { + } else if (item1Type === 'boolean' && item2Type === 'boolean') { + result = item1Value ? 1 : -1; + } else if (item1Type !== item2Type) { + const item1Empty = item1Value === null || item1Value === undefined || item1Value === ''; + const item2Empty = item2Value === null || item2Value === undefined || item2Value === ''; + if (!item1Empty && item2Empty) { result = 1; - } else if (!item1Value && item2Value) { + } else if (item1Empty && !item2Empty) { result = -1; + } else if (!item1Empty && !item2Empty) { + const str1 = String(item1Value).trim(); + const str2 = String(item2Value).trim(); + const num1 = str1.length ? Number(str1) : NaN; + const num2 = str2.length ? Number(str2) : NaN; + if (!isNaN(num1) && !isNaN(num2)) { + result = num1 - num2; + } else { + result = String(item1Value).localeCompare(String(item2Value)); + } } } } diff --git a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts index 38cc0b52f4..94eced7fcf 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts @@ -27,8 +27,10 @@ import { import { additionalMapDataSourcesToDatasources, BaseMapSettings, + latestMapDataLayerTypes, MapDataLayerSettings, MapDataLayerType, + mapDataLayerTypes, MapDataSourceSettings, mapDataSourceSettingsToDatasource, MapType @@ -46,30 +48,21 @@ interface MapDataLayerDsInfo extends AliasFilterPair { type ExportDataSourceInfo = {[dataLayerIndex: number]: MapDataLayerDsInfo}; -interface MapDatasourcesInfo { - trips?: ExportDataSourceInfo; - markers?: ExportDataSourceInfo; - polygons?: ExportDataSourceInfo; - circles?: ExportDataSourceInfo; +type MapDatasourcesInfo = { + [K in MapDataLayerType]?: ExportDataSourceInfo; +} & { additionalDataSources?: ExportDataSourceInfo; -} +}; export const MapModelDefinition: WidgetModelDefinition = { testWidget(widget: Widget): boolean { if (widget?.config?.settings) { const settings = widget.config.settings; if (settings.mapType && [MapType.image, MapType.geoMap].includes(settings.mapType)) { - if (settings.trips && Array.isArray(settings.trips)) { - return true; - } - if (settings.markers && Array.isArray(settings.markers)) { - return true; - } - if (settings.polygons && Array.isArray(settings.polygons)) { - return true; - } - if (settings.circles && Array.isArray(settings.circles)) { - return true; + for (const layerType of mapDataLayerTypes) { + if (Array.isArray(settings[layerType])) { + return true; + } } } } @@ -78,17 +71,11 @@ export const MapModelDefinition: WidgetModelDefinition = { prepareExportInfo(dashboard: Dashboard, widget: Widget): MapDatasourcesInfo { const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; const info: MapDatasourcesInfo = {}; - if (settings.trips?.length) { - info.trips = prepareExportDataSourcesInfo(dashboard, settings.trips); - } - if (settings.markers?.length) { - info.markers = prepareExportDataSourcesInfo(dashboard, settings.markers); - } - if (settings.polygons?.length) { - info.polygons = prepareExportDataSourcesInfo(dashboard, settings.polygons); - } - if (settings.circles?.length) { - info.circles = prepareExportDataSourcesInfo(dashboard, settings.circles); + for (const layerType of mapDataLayerTypes) { + const dataLayerSettings = settings[layerType]; + if (dataLayerSettings?.length) { + info[layerType] = prepareExportDataSourcesInfo(dashboard, dataLayerSettings); + } } if (settings.additionalDataSources?.length) { info.additionalDataSources = prepareExportDataSourcesInfo(dashboard, settings.additionalDataSources); @@ -96,59 +83,36 @@ export const MapModelDefinition: WidgetModelDefinition = { return info; }, updateFromExportInfo(widget: Widget, entityAliases: EntityAliases, filters: Filters, info: MapDatasourcesInfo): void { - const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - if (info?.trips) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.trips, info.trips); - } - if (info?.markers) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.markers, info.markers); - } - if (info?.polygons) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.polygons, info.polygons); - } - if (info?.circles) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.circles, info.circles); - } - if (info?.additionalDataSources) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.additionalDataSources, info.additionalDataSources); + if (info && Object.keys(info).length) { + const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; + for (const layerType of mapDataLayerTypes) { + const layerInfo = info[layerType]; + const dataLayerSettings = settings[layerType]; + if (layerInfo && dataLayerSettings) { + updateMapDatasourceFromExportInfo(entityAliases, filters, dataLayerSettings, layerInfo); + } + } + if (info.additionalDataSources) { + updateMapDatasourceFromExportInfo(entityAliases, filters, settings.additionalDataSources, info.additionalDataSources); + } } }, datasources(widget: Widget): Datasource[] { - const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - const datasources: Datasource[] = []; - if (settings.trips?.length) { - datasources.push(...getMapDataLayersDatasources(settings.trips)); - } - if (settings.markers?.length) { - datasources.push(...getMapDataLayersDatasources(settings.markers)); - } - if (settings.polygons?.length) { - datasources.push(...getMapDataLayersDatasources(settings.polygons)); - } - if (settings.circles?.length) { - datasources.push(...getMapDataLayersDatasources(settings.circles)); - } - if (settings.additionalDataSources?.length) { - datasources.push(...additionalMapDataSourcesToDatasources(settings.additionalDataSources)); - } - return datasources; + return getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, mapDataLayerTypes); }, hasTimewindow(widget: Widget): boolean { const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - if (settings.trips?.length) { + const timeSeriesDataLayerTypes = mapDataLayerTypes.filter(t => !latestMapDataLayerTypes.includes(t)); + if (timeSeriesDataLayerTypes.some(layerType => settings[layerType]?.length)) { return true; - } else { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(settings); - return datasourcesHasAggregation(datasources); } + return datasourcesHasAggregation(getMapDataLayersDatasources(settings, latestMapDataLayerTypes, true)); }, datasourcesHasAggregation(widget: Widget): boolean { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(widget.config.settings as BaseMapSettings); - return datasourcesHasAggregation(datasources); + return datasourcesHasAggregation(getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, latestMapDataLayerTypes, true)); }, datasourcesHasOnlyComparisonAggregation(widget: Widget): boolean { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(widget.config.settings as BaseMapSettings); - return datasourcesHasOnlyComparisonAggregation(datasources); + return datasourcesHasOnlyComparisonAggregation(getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, latestMapDataLayerTypes, true)); } }; @@ -236,7 +200,7 @@ const prepareAliasAndFilterPair = (dashboard: Dashboard, settings: MapDataSource } } -const getMapDataLayersDatasources = (settings: MapDataLayerSettings[], +const getMapDataLayerDatasources = (settings: MapDataLayerSettings[], includeDataKeys = false, dataLayerType?: MapDataLayerType): Datasource[] => { const datasources: Datasource[] = []; settings.forEach((dsSettings) => { @@ -255,16 +219,14 @@ const getMapDataLayersDatasources = (settings: MapDataLayerSettings[], return datasources; }; -const getMapLatestDataLayersDatasources = (settings: BaseMapSettings): Datasource[] => { +const getMapDataLayersDatasources = (settings: BaseMapSettings, + layerTypes: readonly MapDataLayerType[], includeDataKeys = false): Datasource[] => { const datasources: Datasource[] = []; - if (settings.markers?.length) { - datasources.push(...getMapDataLayersDatasources(settings.markers, true, 'markers')); - } - if (settings.polygons?.length) { - datasources.push(...getMapDataLayersDatasources(settings.polygons, true, 'polygons')); - } - if (settings.circles?.length) { - datasources.push(...getMapDataLayersDatasources(settings.circles, true, 'circles')); + for (const layerType of layerTypes) { + const dataLayerSettings = settings[layerType]; + if (dataLayerSettings?.length) { + datasources.push(...getMapDataLayerDatasources(dataLayerSettings, includeDataKeys, layerType)); + } } if (settings.additionalDataSources?.length) { datasources.push(...additionalMapDataSourcesToDatasources(settings.additionalDataSources)); diff --git a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts index 0f1c979425..edfe03cc05 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts @@ -200,9 +200,11 @@ export const defaultBaseDataLayerSettings = (mapType: MapType): Partial { if (!dataLayer.dsType || ![DatasourceType.function, DatasourceType.device, DatasourceType.entity].includes(dataLayer.dsType)) {