Browse Source

Merge remote-tracking branch 'origin/rc' into master-rc-merge

# Conflicts:
#	application/src/main/resources/thingsboard.yml
pull/15586/head
Viacheslav Klimov 3 weeks ago
parent
commit
16cc7d841e
Failed to extract signature
  1. 6
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  2. 131
      application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java
  3. 1
      application/src/main/resources/thingsboard.yml
  4. 105
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  5. 126
      application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java
  6. 84
      application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java
  7. 296
      application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java
  8. 35
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  9. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  10. 2
      pom.xml
  11. 2
      ui-ngx/src/app/shared/import-export/import-export.models.ts
  12. 20
      ui-ngx/src/app/shared/models/page/page-link.ts

6
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -135,6 +135,10 @@ public class InstallScripts {
return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR);
}
public Path getWidgetBundlesDir() {
return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR);
}
public String getDataDir() {
if (!StringUtils.isEmpty(dataDir)) {
if (!Paths.get(this.dataDir).toFile().isDirectory()) {
@ -207,7 +211,7 @@ public class InstallScripts {
public void loadSystemWidgets() {
log.info("Loading system widgets");
Map<Path, JsonNode> widgetsBundlesMap = new HashMap<>();
Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR);
Path widgetBundlesDir = getWidgetBundlesDir();
try (Stream<Path> dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(JSON_EXT))) {
dirStream.forEach(
path -> {

131
application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java

@ -28,8 +28,10 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.install.DatabaseSchemaSettingsService;
import org.thingsboard.server.service.install.InstallScripts;
@ -42,6 +44,9 @@ import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@ -67,6 +72,7 @@ public class SystemPatchApplier {
private final InstallScripts installScripts;
private final DatabaseSchemaSettingsService schemaSettingsService;
private final WidgetTypeService widgetTypeService;
private final WidgetsBundleService widgetsBundleService;
private final ImageService imageService;
@PostConstruct
@ -100,8 +106,11 @@ public class SystemPatchApplier {
updateSqlViews();
log.info("Updated sql database views");
int updated = updateWidgetTypes();
log.info("Updated {} widget types", updated);
WidgetTypeStats widgetStats = updateWidgetTypes();
log.info("System widget types: {} created, {} updated", widgetStats.created(), widgetStats.updated());
int updatedBundles = updateWidgetBundles();
log.info("System widget bundles: {} updated", updatedBundles);
int createdImages = createMissingSystemImages();
log.info("Created {} new system images", createdImages);
@ -171,20 +180,24 @@ public class SystemPatchApplier {
}
}
private int updateWidgetTypes() {
private WidgetTypeStats updateWidgetTypes() {
AtomicInteger created = new AtomicInteger();
AtomicInteger updated = new AtomicInteger();
Path widgetTypesDir = installScripts.getWidgetTypesDir();
if (!Files.exists(widgetTypesDir)) {
log.trace("Widget types directory does not exist: {}", widgetTypesDir);
return 0;
return new WidgetTypeStats(0, 0);
}
try (Stream<Path> dirStream = listDir(widgetTypesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) {
dirStream.forEach(
path -> {
try {
if (updateWidgetTypeFromFile(path)) {
WidgetTypeChange change = updateWidgetTypeFromFile(path);
if (change == WidgetTypeChange.CREATED) {
created.incrementAndGet();
} else if (change == WidgetTypeChange.UPDATED) {
updated.incrementAndGet();
}
} catch (Exception e) {
@ -195,18 +208,26 @@ public class SystemPatchApplier {
);
}
return updated.get();
return new WidgetTypeStats(created.get(), updated.get());
}
private boolean updateWidgetTypeFromFile(Path filePath) {
private WidgetTypeChange updateWidgetTypeFromFile(Path filePath) {
JsonNode json = JacksonUtil.toJsonNode(filePath.toFile());
WidgetTypeDetails fileWidgetType = JacksonUtil.treeToValue(json, WidgetTypeDetails.class);
return saveOrUpdateSystemWidgetType(fileWidgetType);
}
private WidgetTypeChange saveOrUpdateSystemWidgetType(WidgetTypeDetails fileWidgetType) {
String fqn = fileWidgetType.getFqn();
if (fqn == null || fqn.isBlank()) {
throw new RuntimeException("Widget type fqn is missing or blank: " + fileWidgetType.getName());
}
WidgetTypeDetails existingWidgetType = widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, fqn);
if (existingWidgetType == null) {
// We expect only update here, so it's probably never happening, but for test purpose leave it like this:
throw new RuntimeException("Widget type not found: " + fqn);
widgetTypeService.saveWidgetType(fileWidgetType);
log.trace("Created widget type: {}", fqn);
return WidgetTypeChange.CREATED;
}
if (isWidgetTypeChanged(existingWidgetType, fileWidgetType)) {
existingWidgetType.setDescription(fileWidgetType.getDescription());
@ -214,11 +235,95 @@ public class SystemPatchApplier {
existingWidgetType.setDescriptor(fileWidgetType.getDescriptor());
widgetTypeService.saveWidgetType(existingWidgetType);
log.trace("Updated widget type: {}", fqn);
return true;
return WidgetTypeChange.UPDATED;
}
log.trace("Widget type unchanged: {}", fqn);
return false;
return WidgetTypeChange.UNCHANGED;
}
private int updateWidgetBundles() {
AtomicInteger updated = new AtomicInteger();
Path widgetBundlesDir = installScripts.getWidgetBundlesDir();
if (!Files.exists(widgetBundlesDir)) {
log.trace("Widget bundles directory does not exist: {}", widgetBundlesDir);
return 0;
}
try (Stream<Path> dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) {
dirStream.forEach(path -> {
try {
if (processWidgetBundleFile(path)) {
updated.incrementAndGet();
}
} catch (Exception e) {
log.error("Unable to process widgets bundle from json: [{}]", path);
throw new RuntimeException("Unable to process widgets bundle from json", e);
}
});
}
return updated.get();
}
private boolean processWidgetBundleFile(Path filePath) {
JsonNode bundleJson = JacksonUtil.toJsonNode(filePath.toFile());
if (bundleJson == null || !bundleJson.has("widgetsBundle")) {
throw new RuntimeException("Invalid widgets bundle json: " + filePath);
}
WidgetsBundle fileBundle = JacksonUtil.treeToValue(bundleJson.get("widgetsBundle"), WidgetsBundle.class);
String alias = fileBundle.getAlias();
if (alias == null || alias.isBlank()) {
throw new RuntimeException("Widgets bundle alias is missing or blank: " + filePath);
}
WidgetsBundle existingBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, alias);
if (existingBundle == null) {
log.warn("Widgets bundle '{}' not found in DB; bundle creation is not supported by the patch applier, skipping.", alias);
return false;
}
if (bundleJson.has("widgetTypes")) {
throw new RuntimeException("Inline widgetTypes in bundle JSON are not supported by the patch applier; " +
"place widget definitions in widget_types/*.json and reference them via widgetTypeFqns: " + filePath);
}
List<String> fileWidgetFqns = new ArrayList<>();
if (bundleJson.has("widgetTypeFqns")) {
bundleJson.get("widgetTypeFqns").forEach(fqnJson -> fileWidgetFqns.add(fqnJson.asText()));
}
boolean changed = false;
if (isWidgetsBundleChanged(existingBundle, fileBundle)) {
existingBundle.setTitle(fileBundle.getTitle());
existingBundle.setDescription(fileBundle.getDescription());
existingBundle.setImage(fileBundle.getImage());
existingBundle.setOrder(fileBundle.getOrder());
existingBundle.setScada(fileBundle.isScada());
widgetsBundleService.saveWidgetsBundle(existingBundle);
log.trace("Updated widgets bundle metadata: {}", alias);
changed = true;
}
List<String> existingFqns = widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId());
LinkedHashSet<String> mergedFqns = new LinkedHashSet<>(existingFqns);
if (mergedFqns.addAll(fileWidgetFqns)) {
widgetTypeService.updateWidgetsBundleWidgetFqns(TenantId.SYS_TENANT_ID, existingBundle.getId(), new ArrayList<>(mergedFqns));
log.trace("Linked {} new widget fqn(s) to bundle: {}", mergedFqns.size() - existingFqns.size(), alias);
changed = true;
}
return changed;
}
private boolean isWidgetsBundleChanged(WidgetsBundle existing, WidgetsBundle file) {
// Image is intentionally NOT compared: the file always carries a base64 data URI, while the DB stores
// the system-image URL produced by ImageService.replaceBase64WithImageUrl on save. A naive string compare
// would always report a diff and re-save every system bundle on every patch run. Image content changes
// are out of scope for the patch applier — full reinstall covers them.
return !Objects.equals(existing.getTitle(), file.getTitle())
|| !Objects.equals(existing.getDescription(), file.getDescription())
|| !Objects.equals(existing.getOrder(), file.getOrder())
|| existing.isScada() != file.isScada();
}
private int createMissingSystemImages() {
@ -349,4 +454,8 @@ public class SystemPatchApplier {
public record VersionInfo(int major, int minor, int maintenance, int patch) {}
public record WidgetTypeStats(int created, int updated) {}
private enum WidgetTypeChange { CREATED, UPDATED, UNCHANGED }
}

1
application/src/main/resources/thingsboard.yml

@ -477,6 +477,7 @@ sql:
log_tenant_stats: "${SQL_LOG_TENANT_STATS:true}"
# Interval in milliseconds for printing the latest statistic information about the tenant
log_tenant_stats_interval_ms: "${SQL_LOG_TENANT_STATS_INTERVAL_MS:60000}"
entity_data_query_nulls_order_strategy: "${SQL_ENTITY_DATA_QUERY_NULLS_ORDER_STRATEGY:default}" # Nulls ordering strategy for sql entity data query. Possible values: default, nulls_first, nulls_last. The default value is 'default', which means postgres default behavior will be applied: NULLS LAST for ASC and NULLS FIRST for DESC. The 'nulls_first' value means that NULL values will be ordered before non-NULL values regardless of the sort order. The 'nulls_last' value means that NULL values will be ordered after non-NULL values regardless of the sort order.
query:
key-filters-or-conditions:
# Enable/disable OR conditions between key filters in entity data queries

105
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -16,19 +16,27 @@
package org.thingsboard.server.service.entitiy;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.bean.override.mockito.MockitoBean;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.DeviceTypeFilter;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
@ -43,10 +51,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD;
@DaoSqlTest
@TestPropertySource(properties = {
@ -103,6 +114,100 @@ public class EdqsEntityServiceTest extends EntityServiceTest {
assetService.deleteAssetsByTenantId(tenantId);
}
// edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC
@Override
@Test
public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 5; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
devices.add(deviceService.saveDevice(device));
Thread.sleep(1);
}
List<Long> values = List.of(1L, 0L, 0L);
List<ListenableFuture<TimeseriesSaveResult>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i)));
}
Futures.allAsList(timeseriesFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("");
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test"));
EntityDataSortOrder ascSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC);
EntityDataQuery ascQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null);
List<String> ascTelemetry = loadAllData(ascQuery, devices.size()).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "0", "0", "1"));
EntityDataSortOrder descSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC);
EntityDataQuery descQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null);
List<String> descTelemetry = loadAllData(descQuery, devices.size()).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(descTelemetry).containsExactlyElementsOf(List.of("1", "0", "0", "", ""));
}
// edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC
@Override
@Test
public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 5; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
devices.add(deviceService.saveDevice(device));
Thread.sleep(1);
}
List<Boolean> values = List.of(true, false, false);
List<ListenableFuture<TimeseriesSaveResult>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i)));
}
Futures.allAsList(timeseriesFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("");
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test"));
EntityDataSortOrder ascSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC);
EntityDataQuery ascQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null);
List<String> ascTelemetry = loadAllData(ascQuery, devices.size()).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "false", "false", "true"));
EntityDataSortOrder descSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC);
EntityDataQuery descQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null);
List<String> descTelemetry = loadAllData(descQuery, devices.size()).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(descTelemetry).containsExactlyElementsOf(List.of("true", "false", "false", "", ""));
}
@Override
protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) {
return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query),

126
application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

@ -26,6 +26,7 @@ import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.Customer;
@ -47,6 +48,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.AttributesSaveResult;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
@ -79,7 +81,6 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.query.StringFilterPredicate;
import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter;
@ -100,6 +101,7 @@ import org.thingsboard.server.dao.entityview.EntityViewDao;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository;
import org.thingsboard.server.dao.sql.relation.RelationRepository;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
@ -124,7 +126,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.thingsboard.server.common.data.AttributeScope.SERVER_SCOPE;
import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE;
import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD;
import static org.thingsboard.server.common.data.query.EntityKeyType.SERVER_ATTRIBUTE;
@Slf4j
@DaoSqlTest
@ -154,6 +155,8 @@ public class EntityServiceTest extends AbstractControllerTest {
@Autowired
RelationRepository relationRepository;
@Autowired
DefaultEntityQueryRepository entityQueryRepository;
@Autowired
RelationService relationService;
@Autowired
TimeseriesService timeseriesService;
@ -1750,6 +1753,117 @@ public class EntityServiceTest extends AbstractControllerTest {
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException {
try {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 5; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
devices.add(deviceService.saveDevice(device));
Thread.sleep(1);
}
List<Long> values = List.of(1L, 0L, 0L);
List<ListenableFuture<TimeseriesSaveResult>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i)));
}
Futures.allAsList(timeseriesFutures).get();
assertNullsOrdering("default",
List.of("0", "0", "1", "", ""),
List.of("", "", "1", "0", "0"),
devices.size());
assertNullsOrdering("nulls_first",
List.of("", "", "0", "0", "1"),
List.of("", "", "1", "0", "0"),
devices.size());
assertNullsOrdering("nulls_last",
List.of("0", "0", "1", "", ""),
List.of("1", "0", "0", "", ""),
devices.size());
} finally {
deviceService.deleteDevicesByTenantId(tenantId);
}
}
@Test
public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException {
try {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 5; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
devices.add(deviceService.saveDevice(device));
Thread.sleep(1);
}
List<Boolean> values = List.of(true, false, false);
List<ListenableFuture<TimeseriesSaveResult>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < values.size(); i++) {
timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i)));
}
Futures.allAsList(timeseriesFutures).get();
assertNullsOrdering("default",
List.of("false", "false", "true", "", ""),
List.of("", "", "true", "false", "false"),
devices.size());
assertNullsOrdering("nulls_first",
List.of("", "", "false", "false", "true"),
List.of("", "", "true", "false", "false"),
devices.size());
assertNullsOrdering("nulls_last",
List.of("false", "false", "true", "", ""),
List.of("true", "false", "false", "", ""),
devices.size());
} finally {
deviceService.deleteDevicesByTenantId(tenantId);
}
}
private void assertNullsOrdering(String strategy, List<String> expectedAsc, List<String> expectedDesc, int deviceSize) {
String originalStrategy = entityQueryRepository.getNullsOrderStrategy();
ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", strategy);
try {
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("");
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test"));
EntityDataSortOrder ascSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC);
EntityDataQuery ascQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null);
List<String> ascTelemetry = loadAllData(ascQuery, deviceSize).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(ascTelemetry).as("ASC with strategy '%s'", strategy).containsExactlyElementsOf(expectedAsc);
EntityDataSortOrder descSortOrder = new EntityDataSortOrder(
new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC);
EntityDataQuery descQuery = new EntityDataQuery(filter,
new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null);
List<String> descTelemetry = loadAllData(descQuery, deviceSize).stream()
.map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue())
.toList();
assertThat(descTelemetry).as("DESC with strategy '%s'", strategy).containsExactlyElementsOf(expectedDesc);
} finally {
ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", originalStrategy);
}
}
@Test
public void testFindTenantTelemetry() throws ExecutionException, InterruptedException, TimeoutException {
// save timeseries by sys admin
@ -2321,12 +2435,18 @@ public class EntityServiceTest extends AbstractControllerTest {
return timeseriesService.save(tenantId, entityId, timeseries);
}
private ListenableFuture<TimeseriesSaveResult> saveTimeseries(EntityId entityId, String key, Long value) {
protected ListenableFuture<TimeseriesSaveResult> saveTimeseries(EntityId entityId, String key, Long value) {
KvEntry telemetryValue = new LongDataEntry(key, value);
BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue);
return timeseriesService.save(tenantId, entityId, timeseries);
}
protected ListenableFuture<TimeseriesSaveResult> saveTimeseries(EntityId entityId, String key, Boolean value) {
KvEntry telemetryValue = new BooleanDataEntry(key, value);
BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue);
return timeseriesService.save(tenantId, entityId, timeseries);
}
protected void createMultiRootHierarchy(List<Asset> buildings, List<Asset> apartments,
Map<String, Map<UUID, String>> entityNameByTypeMap,
Map<UUID, UUID> childParentRelationMap) throws InterruptedException {

84
application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java

@ -217,6 +217,90 @@ class TbelInvokeServiceTest extends AbstractTbelInvokeTest {
assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull();
}
@Test
void givenForbiddenSocketHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("new java.util.logging.SocketHandler(\"127.0.0.1\", 9999)");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.SocketHandler");
});
}
@Test
void givenForbiddenZipFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("new java.util.zip.ZipFile(\"/tmp/test.zip\")");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.zip.ZipFile");
});
}
@Test
void givenForbiddenFileHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("new java.util.logging.FileHandler(\"/tmp/test.log\")");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.FileHandler");
});
}
@Test
void givenForbiddenJarFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("new java.util.jar.JarFile(\"/tmp/test.jar\")");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.jar.JarFile");
});
}
@Test
void givenForbiddenPreferences_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("java.util.prefs.Preferences.userRoot()");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getMessage()).contains("unresolvable property or identifier: java");
});
}
@Test
void givenForbiddenLocaleServiceProvider_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException {
UUID scriptId = evalScript("new java.util.spi.LocaleServiceProvider()");
assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}"))
.isInstanceOf(ExecutionException.class)
.cause()
.isInstanceOf(TbScriptException.class)
.asInstanceOf(type(TbScriptException.class))
.satisfies(ex -> {
assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME);
assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.spi.LocaleServiceProvider");
});
}
private void assertThatScriptIsBlocked(UUID scriptId) {
assertThatThrownBy(() -> {
invokeScriptResultString(scriptId, "{}");

296
application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java

@ -31,9 +31,12 @@ import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.install.DatabaseSchemaSettingsService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.system.SystemPatchApplier;
@ -41,6 +44,7 @@ import org.thingsboard.server.service.system.SystemPatchApplier;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
@ -81,6 +85,9 @@ public class SystemPatchApplierTest {
@Mock
private WidgetTypeService widgetTypeService;
@Mock
private WidgetsBundleService widgetsBundleService;
@Mock
private ImageService imageService;
@ -155,19 +162,72 @@ public class SystemPatchApplierTest {
}
@Test
void whenWidgetNotFound_thenThrowException() throws Exception {
void whenWidgetNotFound_thenCreateNewWidget() throws Exception {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
WidgetTypeDetails testWidget = createTestWidgetType("test_widget", "Test Widget");
String json = JacksonUtil.toString(testWidget);
WidgetTypeDetails fileWidget = createTestWidgetType("new_widget", "New Widget");
String json = JacksonUtil.toString(fileWidget);
assertNotNull(json);
Files.writeString(widgetTypesDir.resolve("test_widget.json"), json);
Files.writeString(widgetTypesDir.resolve("new_widget.json"), json);
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "new_widget")).thenReturn(null);
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
assertNotNull(stats);
assertEquals(1, stats.created());
assertEquals(0, stats.updated());
verify(widgetTypeService).saveWidgetType(argThat(w -> "new_widget".equals(w.getFqn())));
}
@Test
void whenFqnIsBlank_thenThrowException() throws Exception {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")).thenReturn(null);
WidgetTypeDetails brokenWidget = createTestWidgetType("", "Broken Widget");
String json = JacksonUtil.toString(brokenWidget);
assertNotNull(json);
Files.writeString(widgetTypesDir.resolve("broken.json"), json);
assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"));
verify(widgetTypeService, never()).saveWidgetType(any());
}
@Test
void whenMixOfCreatedAndUpdated_thenStatsAreCorrect() throws Exception {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
WidgetTypeDetails newFileWidget = createTestWidgetType("widget_new", "Widget New");
Files.writeString(widgetTypesDir.resolve("widget_new.json"), JacksonUtil.toString(newFileWidget));
WidgetTypeDetails changedFileWidget = createTestWidgetType("widget_changed", "Widget Changed New Name");
Files.writeString(widgetTypesDir.resolve("widget_changed.json"), JacksonUtil.toString(changedFileWidget));
WidgetTypeDetails sameFileWidget = createTestWidgetType("widget_same", "Widget Same");
Files.writeString(widgetTypesDir.resolve("widget_same.json"), JacksonUtil.toString(sameFileWidget));
WidgetTypeDetails existingChanged = createTestWidgetType("widget_changed", "Widget Changed Old Name");
existingChanged.setId(new WidgetTypeId(UUID.randomUUID()));
WidgetTypeDetails existingSame = createTestWidgetType("widget_same", "Widget Same");
existingSame.setId(new WidgetTypeId(UUID.randomUUID()));
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_new")).thenReturn(null);
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_changed")).thenReturn(existingChanged);
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_same")).thenReturn(existingSame);
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
assertNotNull(stats);
assertEquals(1, stats.created());
assertEquals(1, stats.updated());
verify(widgetTypeService, times(2)).saveWidgetType(any());
}
@Test
@ -189,9 +249,11 @@ public class SystemPatchApplierTest {
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget"))
.thenReturn(existingWidget);
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
assertEquals(1, updated);
assertNotNull(stats);
assertEquals(0, stats.created());
assertEquals(1, stats.updated());
verify(widgetTypeService).saveWidgetType(argThat(w ->
w.getDescriptor().get("version").asInt() == 2
));
@ -214,9 +276,11 @@ public class SystemPatchApplierTest {
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget"))
.thenReturn(existingWidget);
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
assertEquals(1, updated);
assertNotNull(stats);
assertEquals(0, stats.created());
assertEquals(1, stats.updated());
verify(widgetTypeService).saveWidgetType(argThat(w -> "New Name".equals(w.getName())));
}
@ -237,9 +301,11 @@ public class SystemPatchApplierTest {
when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget"))
.thenReturn(existingWidget);
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
assertEquals(0, updated);
assertNotNull(stats);
assertEquals(0, stats.created());
assertEquals(0, stats.updated());
verify(widgetTypeService, never()).saveWidgetType(any());
}
@ -339,8 +405,8 @@ public class SystemPatchApplierTest {
// Simulate work while holding lock
Thread.sleep(100);
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
firstThreadSavedWidget.set(updated != null && updated > 0);
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
firstThreadSavedWidget.set(stats != null && stats.updated() > 0);
ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock");
}
@ -360,8 +426,8 @@ public class SystemPatchApplierTest {
secondThreadAcquiredLock.set(Boolean.TRUE.equals(acquired));
if (secondThreadAcquiredLock.get()) {
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
secondThreadSavedWidget.set(updated != null && updated > 0);
SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes");
secondThreadSavedWidget.set(stats != null && stats.updated() > 0);
ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock");
}
@ -560,6 +626,7 @@ public class SystemPatchApplierTest {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing"));
ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded");
@ -607,6 +674,7 @@ public class SystemPatchApplierTest {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing"));
ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded");
@ -834,6 +902,7 @@ public class SystemPatchApplierTest {
Path widgetTypesDir = tempDir.resolve("widget_types");
Files.createDirectories(widgetTypesDir);
when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing"));
when(imageService.getAllImageKeysByTenantId(TenantId.SYS_TENANT_ID)).thenReturn(Collections.emptySet());
@ -854,4 +923,201 @@ public class SystemPatchApplierTest {
verify(imageService, never()).createOrUpdateSystemImage(anyString(), any(byte[].class));
}
// --- updateWidgetBundles tests ---
@Test
void whenWidgetBundlesDirDoesNotExist_thenReturnsZero() {
when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("missing_bundles"));
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(0, updated);
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
}
@Test
void whenBundleNotInDb_thenSkipWithoutCreation() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"order\":10}," +
"\"widgetTypeFqns\":[\"line_chart\"]}");
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(null);
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(0, updated);
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
}
@Test
void whenBundleExistsAndHasNewFqn_thenMergeFqns() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," +
"\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\",\"new_chart\"]}");
WidgetsBundle existingBundle = createTestBundle("charts", "Charts");
existingBundle.setDescription("d");
existingBundle.setOrder(10);
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle);
when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()))
.thenReturn(List.of("line_chart", "bar_chart"));
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(1, updated);
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
verify(widgetTypeService).updateWidgetsBundleWidgetFqns(
eq(TenantId.SYS_TENANT_ID),
eq(existingBundle.getId()),
argThat(fqns -> fqns.size() == 3
&& fqns.get(0).equals("line_chart")
&& fqns.get(1).equals("bar_chart")
&& fqns.get(2).equals("new_chart"))
);
}
@Test
void whenBundleExistsAndAllFqnsAlreadyLinked_thenNoLinkUpdate() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," +
"\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\"]}");
WidgetsBundle existingBundle = createTestBundle("charts", "Charts");
existingBundle.setDescription("d");
existingBundle.setOrder(10);
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle);
when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()))
.thenReturn(List.of("line_chart", "bar_chart"));
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(0, updated);
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
}
@Test
void whenOnlyBundleImageFormatDiffers_thenNoUpdate() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
// File carries a base64 data URI; DB has the resolved system-image URL — same content, different format.
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10," +
"\"image\":\"data:image/png;base64,iVBORw0KGgo\"}," +
"\"widgetTypeFqns\":[]}");
WidgetsBundle existingBundle = createTestBundle("charts", "Charts");
existingBundle.setDescription("d");
existingBundle.setOrder(10);
existingBundle.setImage("tb-image;/api/images/system/charts.png");
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle);
when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()))
.thenReturn(List.of());
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(0, updated);
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
}
@Test
void whenBundleMetadataChanged_thenUpdateBundle() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"New Title\",\"description\":\"new\",\"order\":20}," +
"\"widgetTypeFqns\":[\"line_chart\"]}");
WidgetsBundle existingBundle = createTestBundle("charts", "Old Title");
existingBundle.setDescription("old");
existingBundle.setOrder(10);
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle);
when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()))
.thenReturn(List.of("line_chart"));
Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles");
assertEquals(1, updated);
verify(widgetsBundleService).saveWidgetsBundle(argThat(b ->
"New Title".equals(b.getTitle()) && "new".equals(b.getDescription()) && b.getOrder() == 20
));
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
}
@Test
void whenBundleAliasIsBlank_thenThrowException() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("broken.json"),
"{\"widgetsBundle\":{\"alias\":\"\",\"title\":\"Broken\"}}");
assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"));
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
}
@Test
void whenBundleJsonMissingWidgetsBundleField_thenThrowException() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("broken.json"), "{\"foo\":\"bar\"}");
assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"));
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
}
@Test
void whenBundleHasInlineWidgetTypes_thenThrowException() throws Exception {
Path bundlesDir = tempDir.resolve("widget_bundles");
Files.createDirectories(bundlesDir);
when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir);
Files.writeString(bundlesDir.resolve("charts.json"),
"{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," +
"\"widgetTypes\":[" +
"{\"fqn\":\"inline_chart\",\"name\":\"Inline\",\"descriptor\":{\"type\":\"latest\"}}" +
"]}");
WidgetsBundle existingBundle = createTestBundle("charts", "Charts");
existingBundle.setDescription("d");
existingBundle.setOrder(10);
when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle);
assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"));
verify(widgetTypeService, never()).saveWidgetType(any());
verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any());
verify(widgetsBundleService, never()).saveWidgetsBundle(any());
}
private WidgetsBundle createTestBundle(String alias, String title) {
WidgetsBundle bundle = new WidgetsBundle();
bundle.setId(new WidgetsBundleId(UUID.randomUUID()));
bundle.setAlias(alias);
bundle.setTitle(title);
bundle.setTenantId(TenantId.SYS_TENANT_ID);
return bundle;
}
}

35
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.sql.query;
import jakarta.annotation.PostConstruct;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@ -62,6 +63,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@ -320,10 +322,19 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
.replace("$in", "from").replace("$out", "to")
.replace("$rootIdCondition", "in (:relation_root_ids)");
private static final String NULLS_ORDER_DEFAULT = "default";
private static final String NULLS_ORDER_FIRST = "nulls_first";
private static final String NULLS_ORDER_LAST = "nulls_last";
private static final Set<String> ACCEPTED_NULLS_ORDER_STRATEGIES = Set.of(NULLS_ORDER_DEFAULT, NULLS_ORDER_FIRST, NULLS_ORDER_LAST);
@Getter
@Value("${sql.relations.max_level:50}")
int maxLevelAllowed; //This value has to be reasonable small to prevent infinite recursion as early as possible
@Getter
@Value("${sql.entity_data_query_nulls_order_strategy:default}")
String nullsOrderStrategy;
private final NamedParameterJdbcTemplate jdbcTemplate;
private final TransactionTemplate transactionTemplate;
private final DefaultQueryLogComponent queryLog;
@ -334,6 +345,15 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
this.queryLog = queryLog;
}
@PostConstruct
void validateNullsOrderStrategy() {
if (!ACCEPTED_NULLS_ORDER_STRATEGIES.contains(nullsOrderStrategy)) {
log.error("Invalid value '{}' for sql.entity_data_query_nulls_order_strategy. Accepted values are: {}. Falling back to '{}'.",
nullsOrderStrategy, ACCEPTED_NULLS_ORDER_STRATEGIES, NULLS_ORDER_DEFAULT);
nullsOrderStrategy = NULLS_ORDER_DEFAULT;
}
}
@Override
public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) {
EntityType entityType = resolveEntityType(query.getEntityFilter());
@ -573,11 +593,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
if (sortOrderMappingOpt.isPresent()) {
EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
String nullsOrder = resolveNullsOrder();
if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
dataQuery = String.format("%s order by %s %s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, direction);
dataQuery = String.format("%s order by %s %s%s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, nullsOrder, direction);
} else {
dataQuery = String.format("%s order by %s %s, %s %s, result.id %s", dataQuery,
sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction, direction);
dataQuery = String.format("%s order by %s %s%s, %s %s, result.id %s", dataQuery,
sortOrderMapping.getSortOrderNumAlias(), direction, nullsOrder, sortOrderMapping.getSortOrderStrAlias(), direction, direction);
}
}
}
@ -596,6 +617,14 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
});
}
private String resolveNullsOrder() {
return switch (nullsOrderStrategy) {
case NULLS_ORDER_FIRST -> " NULLS FIRST";
case NULLS_ORDER_LAST -> " NULLS LAST";
default -> "";
};
}
private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List<EntityKeyMapping> entityFieldsFilters) {
String permissionQuery = this.buildPermissionQuery(ctx, entityFilter);
String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter);

4
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@ -601,8 +601,8 @@ public class EntityKeyMapping {
String attrNumAlias = getSortOrderNumAlias();
String attrVarcharAlias = getSortOrderStrAlias();
String attrSortOrderSelection =
String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," +
"coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v is null then null when %s.bool_v then 1 else 0 end)) %s," +
"coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias);
return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection);
} else {
return String.join(", ", attrValSelection, attrTsSelection);

2
pom.xml

@ -93,7 +93,7 @@
<zookeeper.version>3.9.5</zookeeper.version> <!-- to fix CVE-2026-24308 and CVE-2026-24281. TODO: remove override when fixed in curator-client -->
<protobuf.version>3.25.5</protobuf.version> <!-- A Major v4 does not support by the pubsub yet-->
<grpc.version>1.76.0</grpc.version>
<tbel.version>1.2.9</tbel.version>
<tbel.version>1.2.10</tbel.version>
<lombok.version>1.18.46</lombok.version> <!-- must be in sync with spring-boot-dependencies; needed for maven-compiler-plugin annotationProcessorPaths -->
<paho.client.version>1.2.5</paho.client.version>
<paho.mqttv5.client.version>1.2.5</paho.mqttv5.client.version>

2
ui-ngx/src/app/shared/import-export/import-export.models.ts

@ -210,7 +210,7 @@ function splitCSV(str: string, sep: string): string[] {
foo = foo.shift().split(sep).concat(foo);
}
} else {
foo[x].replace(/""/g, '"');
foo[x] = foo[x].replace(/""/g, '"');
}
}
return foo;

20
ui-ngx/src/app/shared/models/page/page-link.ts

@ -84,11 +84,25 @@ export function sortItems(item1: any, item2: any, property: string, asc: boolean
result = item1Value - item2Value;
} else if (item1Type === 'string' && item2Type === 'string') {
result = item1Value.localeCompare(item2Value);
} else if ((item1Type === 'boolean' && item2Type === 'boolean') || (item1Type !== item2Type)) {
if (item1Value && !item2Value) {
} else if (item1Type === 'boolean' && item2Type === 'boolean') {
result = item1Value ? 1 : -1;
} else if (item1Type !== item2Type) {
const item1Empty = item1Value === null || item1Value === undefined || item1Value === '';
const item2Empty = item2Value === null || item2Value === undefined || item2Value === '';
if (!item1Empty && item2Empty) {
result = 1;
} else if (!item1Value && item2Value) {
} else if (item1Empty && !item2Empty) {
result = -1;
} else if (!item1Empty && !item2Empty) {
const str1 = String(item1Value).trim();
const str2 = String(item2Value).trim();
const num1 = str1.length ? Number(str1) : NaN;
const num2 = str2.length ? Number(str2) : NaN;
if (!isNaN(num1) && !isNaN(num2)) {
result = num1 - num2;
} else {
result = String(item1Value).localeCompare(String(item2Value));
}
}
}
}

Loading…
Cancel
Save