From 4c88676321839442613fa3a425df8933a9578497 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 2 Apr 2026 09:35:00 +0300 Subject: [PATCH 01/26] defer SNMP polling until transport session is fully registered --- .../server/transport/snmp/SnmpTransportContext.java | 3 ++- .../server/transport/snmp/service/SnmpTransportService.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 7d92639f1a..af5e1f04b2 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -160,7 +160,6 @@ public class SnmpTransportContext extends TransportContext { return; } sessions.put(device.getId(), sessionContext); - snmpTransportService.createQueryingTasks(sessionContext); log.info("Established SNMP device session for device {}", device.getId()); } @@ -224,6 +223,8 @@ public class SnmpTransportContext extends TransportContext { registerTransportSession(sessionContext, msg); }); transportService.lifecycleEvent(sessionContext.getTenantId(), sessionContext.getDeviceId(), ComponentLifecycleEvent.STARTED, true, null); + snmpTransportService.createQueryingTasks(sessionContext); + log.info("[{}] Session registered and querying tasks created", sessionContext.getDeviceId()); } else { log.warn("[{}] Failed to process device auth", sessionContext.getDeviceId()); } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 0c39b7e740..5105b35d1d 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -164,7 +164,7 @@ public class SnmpTransportService implements TbTransportService, CommandResponde ScheduledTask scheduledTask = new ScheduledTask(); scheduledTask.init(() -> { try { - if (sessionContext.isActive()) { + if (sessionContext.isActive() && sessionContext.isConnected()) { return sendRequest(sessionContext, repeatingCommunicationConfig); } } catch (Exception e) { @@ -390,7 +390,9 @@ public class SnmpTransportService implements TbTransportService, CommandResponde JsonObject responseData = responseDataMappers.get(requestContext.getCommunicationSpec()).map(response, requestContext); if (responseData.size() == 0) { - log.warn("[{}] No values in the response", sessionContext.getDeviceId()); + log.warn("[{}] No values in the response for spec {}. Response PDUs: {}, Mappings count: {}", + sessionContext.getDeviceId(), requestContext.getCommunicationSpec(), + response, requestContext.getResponseMappings() != null ? requestContext.getResponseMappings().size() : 0); throw new IllegalArgumentException("No values in the response"); } From 1a5007e54df6d719f03305648ff6d7a41e9a5480 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 22 Apr 2026 15:51:39 +0300 Subject: [PATCH 02/26] Fixed telemetry/attribute sorting for rows with null values --- .../src/main/resources/thingsboard.yml | 1 + .../entitiy/EdqsEntityServiceTest.java | 105 +++++++++++++++ .../service/entitiy/EntityServiceTest.java | 126 +++++++++++++++++- .../query/DefaultEntityQueryRepository.java | 19 ++- .../dao/sql/query/EntityKeyMapping.java | 4 +- .../src/app/shared/models/page/page-link.ts | 24 +++- 6 files changed, 266 insertions(+), 13 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 81b56aff28..df949a4b0b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -446,6 +446,7 @@ sql: log_tenant_stats: "${SQL_LOG_TENANT_STATS:true}" # Interval in milliseconds for printing the latest statistic information about the tenant log_tenant_stats_interval_ms: "${SQL_LOG_TENANT_STATS_INTERVAL_MS:60000}" + entity_data_query_nulls_order_strategy: "${SQL_ENTITY_DATA_QUERY_NULLS_ORDER_STRATEGY:default}" # Nulls ordering strategy for sql entity data query. Possible values: default, nulls_first, nulls_last. The default value is 'default', which means postgres default behavior will be applied: NULLS LAST for ASC and NULLS FIRST for DESC. The 'nulls_first' value means that NULL values will be ordered before non-NULL values regardless of the sort order. The 'nulls_last' value means that NULL values will be ordered after non-NULL values regardless of the sort order. postgres: # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}" diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index a16cf99c1d..838695e9a3 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -16,19 +16,27 @@ package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.IdBased; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -43,10 +51,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @DaoSqlTest @TestPropertySource(properties = { @@ -103,6 +114,100 @@ public class EdqsEntityServiceTest extends EntityServiceTest { assetService.deleteAssetsByTenantId(tenantId); } + // edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC + @Override + @Test + public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(1L, 0L, 0L); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "0", "0", "1")); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).containsExactlyElementsOf(List.of("1", "0", "0", "", "")); + } + + // edqs has no nulls order strategies, always returns NULLs first for ASC and NULLs last for DESC + @Override + @Test + public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(true, false, false); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).containsExactlyElementsOf(List.of("", "", "false", "false", "true")); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, devices.size()).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).containsExactlyElementsOf(List.of("true", "false", "false", "", "")); + } + @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 05ab1165f6..b3230b3d45 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; @@ -47,6 +48,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.AttributesSaveResult; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; @@ -79,7 +81,6 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; -import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; @@ -100,6 +101,7 @@ import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -124,7 +126,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.common.data.AttributeScope.SERVER_SCOPE; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; -import static org.thingsboard.server.common.data.query.EntityKeyType.SERVER_ATTRIBUTE; @Slf4j @DaoSqlTest @@ -154,6 +155,8 @@ public class EntityServiceTest extends AbstractControllerTest { @Autowired RelationRepository relationRepository; @Autowired + DefaultEntityQueryRepository entityQueryRepository; + @Autowired RelationService relationService; @Autowired TimeseriesService timeseriesService; @@ -1750,6 +1753,117 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + @Test + public void testSortByNumericTelemetryKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + try { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(1L, 0L, 0L); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + assertNullsOrdering("default", + List.of("0", "0", "1", "", ""), + List.of("", "", "1", "0", "0"), + devices.size()); + + assertNullsOrdering("nulls_first", + List.of("", "", "0", "0", "1"), + List.of("", "", "1", "0", "0"), + devices.size()); + + assertNullsOrdering("nulls_last", + List.of("0", "0", "1", "", ""), + List.of("1", "0", "0", "", ""), + devices.size()); + } finally { + deviceService.deleteDevicesByTenantId(tenantId); + } + } + + @Test + public void testSortByBooleanKeyWithDifferentNullsOrderStrategy() throws ExecutionException, InterruptedException { + try { + List devices = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + Thread.sleep(1); + } + + List values = List.of(true, false, false); + List> timeseriesFutures = new ArrayList<>(); + for (int i = 0; i < values.size(); i++) { + timeseriesFutures.add(saveTimeseries(devices.get(i).getId(), "test", values.get(i))); + } + Futures.allAsList(timeseriesFutures).get(); + + assertNullsOrdering("default", + List.of("false", "false", "true", "", ""), + List.of("", "", "true", "false", "false"), + devices.size()); + + assertNullsOrdering("nulls_first", + List.of("", "", "false", "false", "true"), + List.of("", "", "true", "false", "false"), + devices.size()); + + assertNullsOrdering("nulls_last", + List.of("false", "false", "true", "", ""), + List.of("true", "false", "false", "", ""), + devices.size()); + } finally { + deviceService.deleteDevicesByTenantId(tenantId); + } + } + + private void assertNullsOrdering(String strategy, List expectedAsc, List expectedDesc, int deviceSize) { + String originalStrategy = entityQueryRepository.getNullsOrderStrategy(); + ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", strategy); + try { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(""); + + List entityFields = Collections.singletonList(new EntityKey(ENTITY_FIELD, "name")); + List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "test")); + + EntityDataSortOrder ascSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.ASC); + EntityDataQuery ascQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, ascSortOrder), entityFields, latestValues, null); + List ascTelemetry = loadAllData(ascQuery, deviceSize).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(ascTelemetry).as("ASC with strategy '%s'", strategy).containsExactlyElementsOf(expectedAsc); + + EntityDataSortOrder descSortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.TIME_SERIES, "test"), EntityDataSortOrder.Direction.DESC); + EntityDataQuery descQuery = new EntityDataQuery(filter, + new EntityDataPageLink(10, 0, null, descSortOrder), entityFields, latestValues, null); + List descTelemetry = loadAllData(descQuery, deviceSize).stream() + .map(ed -> ed.getLatest().get(EntityKeyType.TIME_SERIES).get("test").getValue()) + .toList(); + assertThat(descTelemetry).as("DESC with strategy '%s'", strategy).containsExactlyElementsOf(expectedDesc); + } finally { + ReflectionTestUtils.setField(entityQueryRepository, "nullsOrderStrategy", originalStrategy); + } + } + @Test public void testFindTenantTelemetry() throws ExecutionException, InterruptedException, TimeoutException { // save timeseries by sys admin @@ -2321,12 +2435,18 @@ public class EntityServiceTest extends AbstractControllerTest { return timeseriesService.save(tenantId, entityId, timeseries); } - private ListenableFuture saveTimeseries(EntityId entityId, String key, Long value) { + protected ListenableFuture saveTimeseries(EntityId entityId, String key, Long value) { KvEntry telemetryValue = new LongDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); return timeseriesService.save(tenantId, entityId, timeseries); } + protected ListenableFuture saveTimeseries(EntityId entityId, String key, Boolean value) { + KvEntry telemetryValue = new BooleanDataEntry(key, value); + BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); + return timeseriesService.save(tenantId, entityId, timeseries); + } + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index a9b04618c8..134ee1ab34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -322,6 +322,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Value("${sql.relations.max_level:50}") int maxLevelAllowed; //This value has to be reasonable small to prevent infinite recursion as early as possible + @Getter + @Value("${sql.entity_data_query_nulls_order_strategy:default}") + String nullsOrderStrategy; + private final NamedParameterJdbcTemplate jdbcTemplate; private final TransactionTemplate transactionTemplate; private final DefaultQueryLogComponent queryLog; @@ -502,11 +506,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { if (sortOrderMappingOpt.isPresent()) { EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get(); String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc"; + String nullsOrder = resolveNullsOrder(); if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) { - dataQuery = String.format("%s order by %s %s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, direction); + dataQuery = String.format("%s order by %s %s%s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, nullsOrder, direction); } else { - dataQuery = String.format("%s order by %s %s, %s %s, result.id %s", dataQuery, - sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction, direction); + dataQuery = String.format("%s order by %s %s%s, %s %s, result.id %s", dataQuery, + sortOrderMapping.getSortOrderNumAlias(), direction, nullsOrder, sortOrderMapping.getSortOrderStrAlias(), direction, direction); } } } @@ -525,6 +530,14 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } + private String resolveNullsOrder() { + return switch (nullsOrderStrategy) { + case "nulls_first" -> " NULLS FIRST"; + case "nulls_last" -> " NULLS LAST"; + default -> ""; + }; + } + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index c259d48372..496b4eb3bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -494,8 +494,8 @@ public class EntityKeyMapping { String attrNumAlias = getSortOrderNumAlias(); String attrVarcharAlias = getSortOrderStrAlias(); String attrSortOrderSelection = - String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v then 1 else 0 end)) %s," + - "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias); + String.format("coalesce(%s.dbl_v, cast(%s.long_v as double precision), (case when %s.bool_v is null then null when %s.bool_v then 1 else 0 end)) %s," + + "coalesce(%s.str_v, cast(%s.json_v as varchar), '') %s", alias, alias, alias, alias, attrNumAlias, alias, alias, attrVarcharAlias); return String.join(", ", attrValSelection, attrTsSelection, attrSortOrderSelection); } else { return String.join(", ", attrValSelection, attrTsSelection); diff --git a/ui-ngx/src/app/shared/models/page/page-link.ts b/ui-ngx/src/app/shared/models/page/page-link.ts index 023821587b..ede07b3da2 100644 --- a/ui-ngx/src/app/shared/models/page/page-link.ts +++ b/ui-ngx/src/app/shared/models/page/page-link.ts @@ -84,11 +84,25 @@ export function sortItems(item1: any, item2: any, property: string, asc: boolean result = item1Value - item2Value; } else if (item1Type === 'string' && item2Type === 'string') { result = item1Value.localeCompare(item2Value); - } else if ((item1Type === 'boolean' && item2Type === 'boolean') || (item1Type !== item2Type)) { - if (item1Value && !item2Value) { - result = 1; - } else if (!item1Value && item2Value) { - result = -1; + } else if (item1Type === 'boolean' && item2Type === 'boolean') { + result = item1Value ? 1 : -1; + } else if (item1Type !== item2Type) { + const item1Empty = item1Value === null || item1Value === undefined || item1Value === ''; + const item2Empty = item2Value === null || item2Value === undefined || item2Value === ''; + if (!item1Empty && item2Empty) { + return asc ? 1 : -1; + } else if (item1Empty && !item2Empty) { + return asc ? -1 : 1; + } else if (!item1Empty && !item2Empty) { + const str1 = String(item1Value).trim(); + const str2 = String(item2Value).trim(); + const num1 = str1.length ? Number(str1) : NaN; + const num2 = str2.length ? Number(str2) : NaN; + if (!isNaN(num1) && !isNaN(num2)) { + result = num1 - num2; + } else { + result = String(item1Value).localeCompare(String(item2Value)); + } } } } From 43d9832dfeed9149031d10053403f572373767c7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 24 Apr 2026 17:02:48 +0300 Subject: [PATCH 03/26] added validation for nullsOrderStrategy --- .../query/DefaultEntityQueryRepository.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 134ee1ab34..59964d1949 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.query; +import jakarta.annotation.PostConstruct; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -60,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -318,6 +320,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .replace("$in", "from").replace("$out", "to") .replace("$rootIdCondition", "in (:relation_root_ids)"); + private static final String NULLS_ORDER_DEFAULT = "default"; + private static final String NULLS_ORDER_FIRST = "nulls_first"; + private static final String NULLS_ORDER_LAST = "nulls_last"; + private static final Set ACCEPTED_NULLS_ORDER_STRATEGIES = Set.of(NULLS_ORDER_DEFAULT, NULLS_ORDER_FIRST, NULLS_ORDER_LAST); + @Getter @Value("${sql.relations.max_level:50}") int maxLevelAllowed; //This value has to be reasonable small to prevent infinite recursion as early as possible @@ -336,6 +343,15 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { this.queryLog = queryLog; } + @PostConstruct + void validateNullsOrderStrategy() { + if (!ACCEPTED_NULLS_ORDER_STRATEGIES.contains(nullsOrderStrategy)) { + log.error("Invalid value '{}' for sql.entity_data_query_nulls_order_strategy. Accepted values are: {}. Falling back to '{}'.", + nullsOrderStrategy, ACCEPTED_NULLS_ORDER_STRATEGIES, NULLS_ORDER_DEFAULT); + nullsOrderStrategy = NULLS_ORDER_DEFAULT; + } + } + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); @@ -532,8 +548,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private String resolveNullsOrder() { return switch (nullsOrderStrategy) { - case "nulls_first" -> " NULLS FIRST"; - case "nulls_last" -> " NULLS LAST"; + case NULLS_ORDER_FIRST -> " NULLS FIRST"; + case NULLS_ORDER_LAST -> " NULLS LAST"; default -> ""; }; } From 517242c9d47b056a93452cad139f90be1fe616c2 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 29 Apr 2026 12:03:49 +0300 Subject: [PATCH 04/26] fixed ui sorting --- ui-ngx/src/app/shared/models/page/page-link.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/shared/models/page/page-link.ts b/ui-ngx/src/app/shared/models/page/page-link.ts index ede07b3da2..eb2c059c2a 100644 --- a/ui-ngx/src/app/shared/models/page/page-link.ts +++ b/ui-ngx/src/app/shared/models/page/page-link.ts @@ -90,9 +90,9 @@ export function sortItems(item1: any, item2: any, property: string, asc: boolean const item1Empty = item1Value === null || item1Value === undefined || item1Value === ''; const item2Empty = item2Value === null || item2Value === undefined || item2Value === ''; if (!item1Empty && item2Empty) { - return asc ? 1 : -1; + result = 1; } else if (item1Empty && !item2Empty) { - return asc ? -1 : 1; + result = -1; } else if (!item1Empty && !item2Empty) { const str1 = String(item1Value).trim(); const str2 = String(item2Value).trim(); From 5dff45403891d9882421c644ab4b08b500270b44 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 1 May 2026 11:37:33 +0300 Subject: [PATCH 05/26] fix(queue): prevent consumer busy-wait when partition assignment is empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AbstractTbQueueConsumerTemplate.poll() returned emptyList() immediately when partitions was empty, bypassing both doPoll() and the secondary sleep guard (which is also skipped for backends that report isLongPollingSupported()==true, e.g. Kafka). The result was a permanent CPU-burning loop on consumers whose partition assignment ended up empty after a rebalance cascade — observed on 26 ie-downlink-consumer threads (~244% total CPU) until container restart. Route the empty-partition path through sleepAndReturnEmpty() so the caller honors durationInMillis regardless of long-polling support. --- .../AbstractTbQueueConsumerTemplate.java | 5 +- .../AbstractTbQueueConsumerTemplateTest.java | 146 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 07726f0254..7f12f6aab5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -98,7 +98,10 @@ public abstract class AbstractTbQueueConsumerTemplate i doSubscribe(partitions); subscribed = true; } - records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); + if (partitions.isEmpty()) { + return sleepAndReturnEmpty(startNanos, durationInMillis); + } + records = doPoll(durationInMillis); } finally { consumerLock.unlock(); } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java new file mode 100644 index 0000000000..a016408ad4 --- /dev/null +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplateTest.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.BDDMockito.never; +import static org.mockito.BDDMockito.spy; +import static org.mockito.BDDMockito.times; +import static org.mockito.BDDMockito.verify; + +@Slf4j +@ExtendWith(MockitoExtension.class) +public class AbstractTbQueueConsumerTemplateTest { + + private static final long POLL_DURATION_MS = 100L; + private static final long SLEEP_TOLERANCE_MS = 20L; + + @Test + public void givenEmptyPartitionsAndLongPollingSupported_whenPoll_thenSleepsAndDoesNotCallDoPoll() { + // Regression: with empty partitions AND isLongPollingSupported()==true (e.g. Kafka), + // poll() previously returned instantly with no sleep, causing the consumer loop to busy-spin. + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, never()).doPoll(anyLong()); + assertThat("poll() must sleep ~durationInMillis when partitions are empty (no busy-wait)", + elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + @Test + public void givenEmptyPartitionsAndNoLongPolling_whenPoll_thenSleepsAndDoesNotCallDoPoll() { + TestConsumer consumer = spy(new TestConsumer("test-topic", false)); + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, never()).doPoll(anyLong()); + assertThat(elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + @Test + public void givenNonEmptyPartitions_whenPoll_thenCallsDoPoll() { + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.singleton(new TopicPartitionInfo("test-topic", null, 0, true))); + + List result = consumer.poll(POLL_DURATION_MS); + + assertThat(result, is(empty())); + verify(consumer, times(1)).doPoll(POLL_DURATION_MS); + } + + @Test + public void givenPartitionsBecomeEmptyAfterRebalance_whenPollAgain_thenStopsCallingDoPoll() { + // Reproduces the observed trigger: a rebalance leaves the consumer with an empty + // partition assignment. Subsequent poll() calls must not busy-spin or call doPoll(). + TestConsumer consumer = spy(new TestConsumer("test-topic", true)); + consumer.subscribe(Collections.singleton(new TopicPartitionInfo("test-topic", null, 0, true))); + consumer.poll(POLL_DURATION_MS); + verify(consumer, times(1)).doPoll(POLL_DURATION_MS); + + consumer.subscribe(Collections.emptySet()); + + long startNs = System.nanoTime(); + List result = consumer.poll(POLL_DURATION_MS); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs); + + assertThat(result, is(empty())); + verify(consumer, times(1)).doPoll(anyLong()); + assertThat(elapsedMs, greaterThanOrEqualTo(POLL_DURATION_MS - SLEEP_TOLERANCE_MS)); + } + + static class TestConsumer extends AbstractTbQueueConsumerTemplate { + + private final boolean longPollingSupported; + + TestConsumer(String topic, boolean longPollingSupported) { + super(topic); + this.longPollingSupported = longPollingSupported; + } + + @Override + protected List doPoll(long durationInMillis) { + return Collections.emptyList(); + } + + @Override + protected TbQueueMsg decode(Object record) { + return null; + } + + @Override + protected void doSubscribe(Set partitions) { + } + + @Override + protected void doCommit() { + } + + @Override + protected void doUnsubscribe() { + } + + @Override + protected boolean isLongPollingSupported() { + return longPollingSupported; + } + } + +} From 704064615a42617e61e10a468cbabf31c10f7260 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 5 May 2026 11:55:26 +0300 Subject: [PATCH 06/26] feat(js-executor): support lz4 compression for Kafka producer --- msa/js-executor/config/default.yml | 2 +- msa/js-executor/package.json | 1 + msa/js-executor/queue/kafkaTemplate.ts | 17 +++- msa/js-executor/yarn.lock | 105 +++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index be49ece5a6..5939b0e29a 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -34,7 +34,7 @@ kafka: partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 connectionTimeout: "1000" # The default value in kafkajs is: 1000 - compression: "none" # gzip or uncompressed + compression: "none" # gzip, lz4 or none topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" use_confluent_cloud: false client_id: "kafkajs" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index b096a3ae99..dfa50f6765 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -17,6 +17,7 @@ "express": "^5.1.0", "js-yaml": "^4.1.1", "kafkajs": "^2.2.4", + "@2l/kafkajs-lz4": "^1.3.2", "long": "^5.3.2", "uuid-parse": "^1.1.0", "winston": "^3.17.0", diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 4527e9e1ce..5a340b2cfd 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -21,6 +21,7 @@ import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' import { IQueue } from './queue.models'; import { Admin, + CompressionCodecs, CompressionTypes, Consumer, Kafka, @@ -30,11 +31,14 @@ import { Producer, TopicMessages } from 'kafkajs'; +import LZ4Codec from '@2l/kafkajs-lz4'; import { isNotEmptyStr } from '../api/utils'; import { KeyObject } from 'tls'; import process, { exit, kill } from 'process'; +CompressionCodecs[CompressionTypes.LZ4] = new LZ4Codec().codec; + export class KafkaTemplate implements IQueue { private logger = _logger(`kafkaTemplate`); @@ -46,7 +50,18 @@ export class KafkaTemplate implements IQueue { private linger = Number(config.get('kafka.linger_ms')); private requestTimeout = Number(config.get('kafka.requestTimeout')); private connectionTimeout = Number(config.get('kafka.connectionTimeout')); - private compressionType = (config.get('kafka.compression') === "gzip") ? CompressionTypes.GZIP : CompressionTypes.None; + private compressionType = KafkaTemplate.resolveCompressionType(config.get('kafka.compression')); + + private static resolveCompressionType(compression: string): CompressionTypes { + switch (compression) { + case 'gzip': + return CompressionTypes.GZIP; + case 'lz4': + return CompressionTypes.LZ4; + default: + return CompressionTypes.None; + } + } private partitionsConsumedConcurrently = Number(config.get('kafka.partitions_consumed_concurrently')); private kafkaClient: Kafka; diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 7e2fe58a13..5e3330e0d7 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -2,6 +2,78 @@ # yarn lockfile v1 +"@2l/kafkajs-lz4@^1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@2l/kafkajs-lz4/-/kafkajs-lz4-1.3.2.tgz#25259f693a709816ef4eb62514c1ce1bab80502f" + integrity sha512-yq5dx4CbL2sofWXKuadyty3ZKRcZyqe5iiuuOWjZgtIed9MBZ2Wqe5hQN18+a5XVfR4SFQGVISb/a0oTjdPjwQ== + dependencies: + lz4-napi "^2.8.0" + +"@antoniomuso/lz4-napi-android-arm-eabi@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-android-arm-eabi/-/lz4-napi-android-arm-eabi-2.9.0.tgz#5ea67847f0a761c9aec22a31212d0e429ca01fb2" + integrity sha512-aeT/9SoWq7rnmzssWuCKUPaxVt3fzE9q+xq/ZHbnUSmrm8/EhLOACMvQeCOnL0IZsmPh8EpuwIE1TZyM9iQPRA== + +"@antoniomuso/lz4-napi-android-arm64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-android-arm64/-/lz4-napi-android-arm64-2.9.0.tgz#64cb0aac70267bb071bba2d8e382301e058811ca" + integrity sha512-ibQ0qiEvmljXAM97IgOZfh+PeiSQ0Rqf2HErJlZPVm2v4GVJoB67v21v1TUydqNNV5L8bwufVoZ90nheL8X9ZA== + +"@antoniomuso/lz4-napi-darwin-arm64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-darwin-arm64/-/lz4-napi-darwin-arm64-2.9.0.tgz#669e48b165af11cec20581acc2caa0d1c0f84472" + integrity sha512-1su4K1MWa4bcWoZlHajv+luGmFDV1JwIsvjtDF+0HhUveSDPP+8A4Z34zOZidURIr08Sl7M7ViPth6ZQ9SqnAA== + +"@antoniomuso/lz4-napi-darwin-x64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-darwin-x64/-/lz4-napi-darwin-x64-2.9.0.tgz#a00ddf021772e26bd01b3ca5e3025f11d1667edf" + integrity sha512-8Lnbm2MkdJtiJ/nbcRS9zRyGp3G0sG6D+Y/x1vTP8nZs3/f8tBwYNsjxCQyyXNNyHcYWwVGbk68onP/pyDljOA== + +"@antoniomuso/lz4-napi-freebsd-x64@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-freebsd-x64/-/lz4-napi-freebsd-x64-2.9.0.tgz#fdc2b292489bd6d4e44eb7bb059fb9bd4b860f14" + integrity sha512-k04EMVOjntKDPrdR4Tf8WyNseuk9PTtSGw8WHyp4CTjoR1s+YJxtp9SMnThe5o2q0TATwk8WGYb/Howrp5OMxw== + +"@antoniomuso/lz4-napi-linux-arm-gnueabihf@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm-gnueabihf/-/lz4-napi-linux-arm-gnueabihf-2.9.0.tgz#9fcdf93b3ca5aa24469e4bf11dd7100df8c71975" + integrity sha512-H92F8zPZmgy2r8IhCWh3qIBfLp2BQ5cp18RoDXhtGFWwkh+5gVWrZp11IVznrsdgB0QeW0VR7dAMMHg3WLOPfA== + +"@antoniomuso/lz4-napi-linux-arm64-gnu@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm64-gnu/-/lz4-napi-linux-arm64-gnu-2.9.0.tgz#9d22e075a9cb60c3cb415428138618ae4a79f3c5" + integrity sha512-25crh0qs/3Rj3fMI8ulYD0DoaKsidUhMBki2aeO69ZK+F8bmQ/e2++FlgJ6f3EgMP5CNxJtnZXKhPOraQWjwAw== + +"@antoniomuso/lz4-napi-linux-arm64-musl@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-arm64-musl/-/lz4-napi-linux-arm64-musl-2.9.0.tgz#d957bce3accd49199bb6de522f5163efa2068fe4" + integrity sha512-eJtHp38zuLaYI0/cOV/BKcNQiXUBo4GPx53FTf0Y307yUjLsn48LNeN0vD28Ct9YrbUae3bQvMD5AD86She0ww== + +"@antoniomuso/lz4-napi-linux-x64-gnu@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-x64-gnu/-/lz4-napi-linux-x64-gnu-2.9.0.tgz#c86567d5aa23863059afbee829fb9f45895d3869" + integrity sha512-mDjS4dyjRKaZQcAP71SphkYH5r3kufB30ih/VETVu/br2toCfBk6Zr1xhL1r+L7FaVAFzF62B7h30CiqrN0Awg== + +"@antoniomuso/lz4-napi-linux-x64-musl@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-linux-x64-musl/-/lz4-napi-linux-x64-musl-2.9.0.tgz#7d02612dec7d6247645aa321871b15247da0ce7d" + integrity sha512-pvU7Z7qjkjn17NkddBtBQ7C2iRqjtZ7WJ3Jqrjtj4XxolY3Q0HaYMvWjkWhzb9AKGZbj5y+EHYtbVoZJ2TSQhQ== + +"@antoniomuso/lz4-napi-win32-arm64-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-arm64-msvc/-/lz4-napi-win32-arm64-msvc-2.9.0.tgz#c901bfec718303ed86867129ecc0371eaa883517" + integrity sha512-aioLlbpJl0QPEXLXhh2bzyitc3T7Jot3f1ap6WdKiRa+CIjMHXw1nxJXy07MLXif10r+qVZr86ic8dvwErgqEQ== + +"@antoniomuso/lz4-napi-win32-ia32-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-ia32-msvc/-/lz4-napi-win32-ia32-msvc-2.9.0.tgz#11e48b18b7923c250735e0130998e5968ae91130" + integrity sha512-VaF4XMTdYb59TsPsiqnWwsNaWKHhgxF33z5p4zg4n0tp20eWozl76hn8B+aXthSs40W0W1N97QhxxV4oXGd8cg== + +"@antoniomuso/lz4-napi-win32-x64-msvc@2.9.0": + version "2.9.0" + resolved "https://registry.yarnpkg.com/@antoniomuso/lz4-napi-win32-x64-msvc/-/lz4-napi-win32-x64-msvc-2.9.0.tgz#20d9f71a638f3277cd0b7662e966f90e53d98af8" + integrity sha512-wfA8ShO3eGLxJ1LDwXJo87XL2D4NkMJV1pfHPvLZpD0MWb9u8VfgS+gKK5YhT7XKjzVdeIna9jgFdn2HBnZBxA== + "@babel/generator@^7.23.0": version "7.28.3" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.28.3.tgz#9626c1741c650cbac39121694a0f2d7451b8ef3e" @@ -100,6 +172,18 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@napi-rs/triples@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@napi-rs/triples/-/triples-1.2.0.tgz#bcd9c936acb93890e7015818e0181f3db421aafa" + integrity sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA== + +"@node-rs/helper@^1.3.3": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@node-rs/helper/-/helper-1.6.0.tgz#83e5381de6e898d0b8c92178bb8d897d619e3a3a" + integrity sha512-2OTh/tokcLA1qom1zuCJm2gQzaZljCCbtX1YCrwRVd/toz7KxaDRFeLTAPwhs8m9hWgzrBn5rShRm6IaZofCPw== + dependencies: + "@napi-rs/triples" "^1.2.0" + "@tsconfig/node10@^1.0.7": version "1.0.11" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" @@ -998,6 +1082,27 @@ long@^5.3.2: resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== +lz4-napi@^2.8.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/lz4-napi/-/lz4-napi-2.9.0.tgz#4a700974a1154f82b3c0e1b7030fe63140051e7d" + integrity sha512-ZOWqxBMIK5768aD20tYn5B6Pp9WPM9UG/LHk8neG9p0gC1DtjdzhTtlkxhAjvTRpmJvMtnnqLKlT+COlqAt9cQ== + dependencies: + "@node-rs/helper" "^1.3.3" + optionalDependencies: + "@antoniomuso/lz4-napi-android-arm-eabi" "2.9.0" + "@antoniomuso/lz4-napi-android-arm64" "2.9.0" + "@antoniomuso/lz4-napi-darwin-arm64" "2.9.0" + "@antoniomuso/lz4-napi-darwin-x64" "2.9.0" + "@antoniomuso/lz4-napi-freebsd-x64" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm-gnueabihf" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm64-gnu" "2.9.0" + "@antoniomuso/lz4-napi-linux-arm64-musl" "2.9.0" + "@antoniomuso/lz4-napi-linux-x64-gnu" "2.9.0" + "@antoniomuso/lz4-napi-linux-x64-musl" "2.9.0" + "@antoniomuso/lz4-napi-win32-arm64-msvc" "2.9.0" + "@antoniomuso/lz4-napi-win32-ia32-msvc" "2.9.0" + "@antoniomuso/lz4-napi-win32-x64-msvc" "2.9.0" + make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" From ab578a5f87e5f90595937f85960ab9e142c77806 Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Tue, 5 May 2026 13:09:32 +0200 Subject: [PATCH 07/26] Added support for new system widgets in patch upgrades SystemPatchApplier now creates widget types missing from the DB instead of throwing, and merges new fqns into existing system bundles. Bundle creation and inline widgetTypes in bundle JSONs are explicitly rejected as out-of-scope for patch upgrades. --- .../service/install/InstallScripts.java | 6 +- .../service/system/SystemPatchApplier.java | 128 ++++++++- .../server/system/SystemPatchApplierTest.java | 269 +++++++++++++++++- 3 files changed, 376 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 364fcb47dd..c7203a0a07 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -135,6 +135,10 @@ public class InstallScripts { return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR); } + public Path getWidgetBundlesDir() { + return Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); + } + public String getDataDir() { if (!StringUtils.isEmpty(dataDir)) { if (!Paths.get(this.dataDir).toFile().isDirectory()) { @@ -207,7 +211,7 @@ public class InstallScripts { public void loadSystemWidgets() { log.info("Loading system widgets"); Map widgetsBundlesMap = new HashMap<>(); - Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); + Path widgetBundlesDir = getWidgetBundlesDir(); try (Stream dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(JSON_EXT))) { dirStream.forEach( path -> { diff --git a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java index bb5052fd76..e5bc65a047 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java +++ b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java @@ -28,8 +28,10 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.DatabaseSchemaSettingsService; import org.thingsboard.server.service.install.InstallScripts; @@ -42,6 +44,9 @@ import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -67,6 +72,7 @@ public class SystemPatchApplier { private final InstallScripts installScripts; private final DatabaseSchemaSettingsService schemaSettingsService; private final WidgetTypeService widgetTypeService; + private final WidgetsBundleService widgetsBundleService; private final ImageService imageService; @PostConstruct @@ -100,8 +106,11 @@ public class SystemPatchApplier { updateSqlViews(); log.info("Updated sql database views"); - int updated = updateWidgetTypes(); - log.info("Updated {} widget types", updated); + WidgetTypeStats widgetStats = updateWidgetTypes(); + log.info("System widget types: {} created, {} updated", widgetStats.created(), widgetStats.updated()); + + int updatedBundles = updateWidgetBundles(); + log.info("System widget bundles: {} updated", updatedBundles); int createdImages = createMissingSystemImages(); log.info("Created {} new system images", createdImages); @@ -171,20 +180,24 @@ public class SystemPatchApplier { } } - private int updateWidgetTypes() { + private WidgetTypeStats updateWidgetTypes() { + AtomicInteger created = new AtomicInteger(); AtomicInteger updated = new AtomicInteger(); Path widgetTypesDir = installScripts.getWidgetTypesDir(); if (!Files.exists(widgetTypesDir)) { log.trace("Widget types directory does not exist: {}", widgetTypesDir); - return 0; + return new WidgetTypeStats(0, 0); } try (Stream dirStream = listDir(widgetTypesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) { dirStream.forEach( path -> { try { - if (updateWidgetTypeFromFile(path)) { + WidgetTypeChange change = updateWidgetTypeFromFile(path); + if (change == WidgetTypeChange.CREATED) { + created.incrementAndGet(); + } else if (change == WidgetTypeChange.UPDATED) { updated.incrementAndGet(); } } catch (Exception e) { @@ -195,18 +208,26 @@ public class SystemPatchApplier { ); } - return updated.get(); + return new WidgetTypeStats(created.get(), updated.get()); } - private boolean updateWidgetTypeFromFile(Path filePath) { + private WidgetTypeChange updateWidgetTypeFromFile(Path filePath) { JsonNode json = JacksonUtil.toJsonNode(filePath.toFile()); WidgetTypeDetails fileWidgetType = JacksonUtil.treeToValue(json, WidgetTypeDetails.class); + return saveOrUpdateSystemWidgetType(fileWidgetType); + } + + private WidgetTypeChange saveOrUpdateSystemWidgetType(WidgetTypeDetails fileWidgetType) { String fqn = fileWidgetType.getFqn(); + if (fqn == null || fqn.isBlank()) { + throw new RuntimeException("Widget type fqn is missing or blank: " + fileWidgetType.getName()); + } WidgetTypeDetails existingWidgetType = widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, fqn); if (existingWidgetType == null) { - // We expect only update here, so it's probably never happening, but for test purpose leave it like this: - throw new RuntimeException("Widget type not found: " + fqn); + widgetTypeService.saveWidgetType(fileWidgetType); + log.trace("Created widget type: {}", fqn); + return WidgetTypeChange.CREATED; } if (isWidgetTypeChanged(existingWidgetType, fileWidgetType)) { existingWidgetType.setDescription(fileWidgetType.getDescription()); @@ -214,11 +235,92 @@ public class SystemPatchApplier { existingWidgetType.setDescriptor(fileWidgetType.getDescriptor()); widgetTypeService.saveWidgetType(existingWidgetType); log.trace("Updated widget type: {}", fqn); - return true; + return WidgetTypeChange.UPDATED; } log.trace("Widget type unchanged: {}", fqn); - return false; + return WidgetTypeChange.UNCHANGED; + } + + private int updateWidgetBundles() { + AtomicInteger updated = new AtomicInteger(); + Path widgetBundlesDir = installScripts.getWidgetBundlesDir(); + + if (!Files.exists(widgetBundlesDir)) { + log.trace("Widget bundles directory does not exist: {}", widgetBundlesDir); + return 0; + } + + try (Stream dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) { + dirStream.forEach(path -> { + try { + if (processWidgetBundleFile(path)) { + updated.incrementAndGet(); + } + } catch (Exception e) { + log.error("Unable to process widgets bundle from json: [{}]", path); + throw new RuntimeException("Unable to process widgets bundle from json", e); + } + }); + } + + return updated.get(); + } + + private boolean processWidgetBundleFile(Path filePath) { + JsonNode bundleJson = JacksonUtil.toJsonNode(filePath.toFile()); + if (bundleJson == null || !bundleJson.has("widgetsBundle")) { + throw new RuntimeException("Invalid widgets bundle json: " + filePath); + } + WidgetsBundle fileBundle = JacksonUtil.treeToValue(bundleJson.get("widgetsBundle"), WidgetsBundle.class); + String alias = fileBundle.getAlias(); + if (alias == null || alias.isBlank()) { + throw new RuntimeException("Widgets bundle alias is missing or blank: " + filePath); + } + + WidgetsBundle existingBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, alias); + if (existingBundle == null) { + log.warn("Widgets bundle '{}' not found in DB; bundle creation is not supported by the patch applier, skipping.", alias); + return false; + } + + if (bundleJson.has("widgetTypes")) { + throw new RuntimeException("Inline widgetTypes in bundle JSON are not supported by the patch applier; " + + "place widget definitions in widget_types/*.json and reference them via widgetTypeFqns: " + filePath); + } + List fileWidgetFqns = new ArrayList<>(); + if (bundleJson.has("widgetTypeFqns")) { + bundleJson.get("widgetTypeFqns").forEach(fqnJson -> fileWidgetFqns.add(fqnJson.asText())); + } + + boolean changed = false; + if (isWidgetsBundleChanged(existingBundle, fileBundle)) { + existingBundle.setTitle(fileBundle.getTitle()); + existingBundle.setDescription(fileBundle.getDescription()); + existingBundle.setImage(fileBundle.getImage()); + existingBundle.setOrder(fileBundle.getOrder()); + existingBundle.setScada(fileBundle.isScada()); + widgetsBundleService.saveWidgetsBundle(existingBundle); + log.trace("Updated widgets bundle metadata: {}", alias); + changed = true; + } + + List existingFqns = widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId()); + LinkedHashSet mergedFqns = new LinkedHashSet<>(existingFqns); + if (mergedFqns.addAll(fileWidgetFqns)) { + widgetTypeService.updateWidgetsBundleWidgetFqns(TenantId.SYS_TENANT_ID, existingBundle.getId(), new ArrayList<>(mergedFqns)); + log.trace("Linked {} new widget fqn(s) to bundle: {}", mergedFqns.size() - existingFqns.size(), alias); + changed = true; + } + return changed; + } + + private boolean isWidgetsBundleChanged(WidgetsBundle existing, WidgetsBundle file) { + return !Objects.equals(existing.getTitle(), file.getTitle()) + || !Objects.equals(existing.getDescription(), file.getDescription()) + || !Objects.equals(existing.getImage(), file.getImage()) + || !Objects.equals(existing.getOrder(), file.getOrder()) + || existing.isScada() != file.isScada(); } private int createMissingSystemImages() { @@ -349,4 +451,8 @@ public class SystemPatchApplier { public record VersionInfo(int major, int minor, int maintenance, int patch) {} + public record WidgetTypeStats(int created, int updated) {} + + private enum WidgetTypeChange { CREATED, UPDATED, UNCHANGED } + } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java index 51e35bd999..4275b27560 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java @@ -31,9 +31,12 @@ import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.service.install.DatabaseSchemaSettingsService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.system.SystemPatchApplier; @@ -41,6 +44,7 @@ import org.thingsboard.server.service.system.SystemPatchApplier; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -81,6 +85,9 @@ public class SystemPatchApplierTest { @Mock private WidgetTypeService widgetTypeService; + @Mock + private WidgetsBundleService widgetsBundleService; + @Mock private ImageService imageService; @@ -155,19 +162,72 @@ public class SystemPatchApplierTest { } @Test - void whenWidgetNotFound_thenThrowException() throws Exception { + void whenWidgetNotFound_thenCreateNewWidget() throws Exception { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); - WidgetTypeDetails testWidget = createTestWidgetType("test_widget", "Test Widget"); - String json = JacksonUtil.toString(testWidget); + WidgetTypeDetails fileWidget = createTestWidgetType("new_widget", "New Widget"); + String json = JacksonUtil.toString(fileWidget); assertNotNull(json); - Files.writeString(widgetTypesDir.resolve("test_widget.json"), json); + Files.writeString(widgetTypesDir.resolve("new_widget.json"), json); + + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "new_widget")).thenReturn(null); + + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")).thenReturn(null); + assertNotNull(stats); + assertEquals(1, stats.created()); + assertEquals(0, stats.updated()); + verify(widgetTypeService).saveWidgetType(argThat(w -> "new_widget".equals(w.getFqn()))); + } + + @Test + void whenFqnIsBlank_thenThrowException() throws Exception { + Path widgetTypesDir = tempDir.resolve("widget_types"); + Files.createDirectories(widgetTypesDir); + when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + + WidgetTypeDetails brokenWidget = createTestWidgetType("", "Broken Widget"); + String json = JacksonUtil.toString(brokenWidget); + assertNotNull(json); + Files.writeString(widgetTypesDir.resolve("broken.json"), json); assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes")); + verify(widgetTypeService, never()).saveWidgetType(any()); + } + + @Test + void whenMixOfCreatedAndUpdated_thenStatsAreCorrect() throws Exception { + Path widgetTypesDir = tempDir.resolve("widget_types"); + Files.createDirectories(widgetTypesDir); + when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + + WidgetTypeDetails newFileWidget = createTestWidgetType("widget_new", "Widget New"); + Files.writeString(widgetTypesDir.resolve("widget_new.json"), JacksonUtil.toString(newFileWidget)); + + WidgetTypeDetails changedFileWidget = createTestWidgetType("widget_changed", "Widget Changed New Name"); + Files.writeString(widgetTypesDir.resolve("widget_changed.json"), JacksonUtil.toString(changedFileWidget)); + + WidgetTypeDetails sameFileWidget = createTestWidgetType("widget_same", "Widget Same"); + Files.writeString(widgetTypesDir.resolve("widget_same.json"), JacksonUtil.toString(sameFileWidget)); + + WidgetTypeDetails existingChanged = createTestWidgetType("widget_changed", "Widget Changed Old Name"); + existingChanged.setId(new WidgetTypeId(UUID.randomUUID())); + + WidgetTypeDetails existingSame = createTestWidgetType("widget_same", "Widget Same"); + existingSame.setId(new WidgetTypeId(UUID.randomUUID())); + + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_new")).thenReturn(null); + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_changed")).thenReturn(existingChanged); + when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "widget_same")).thenReturn(existingSame); + + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + + assertNotNull(stats); + assertEquals(1, stats.created()); + assertEquals(1, stats.updated()); + verify(widgetTypeService, times(2)).saveWidgetType(any()); } @Test @@ -189,9 +249,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(1, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(1, stats.updated()); verify(widgetTypeService).saveWidgetType(argThat(w -> w.getDescriptor().get("version").asInt() == 2 )); @@ -214,9 +276,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(1, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(1, stats.updated()); verify(widgetTypeService).saveWidgetType(argThat(w -> "New Name".equals(w.getName()))); } @@ -237,9 +301,11 @@ public class SystemPatchApplierTest { when(widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "test_widget")) .thenReturn(existingWidget); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - assertEquals(0, updated); + assertNotNull(stats); + assertEquals(0, stats.created()); + assertEquals(0, stats.updated()); verify(widgetTypeService, never()).saveWidgetType(any()); } @@ -339,8 +405,8 @@ public class SystemPatchApplierTest { // Simulate work while holding lock Thread.sleep(100); - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - firstThreadSavedWidget.set(updated != null && updated > 0); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + firstThreadSavedWidget.set(stats != null && stats.updated() > 0); ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock"); } @@ -360,8 +426,8 @@ public class SystemPatchApplierTest { secondThreadAcquiredLock.set(Boolean.TRUE.equals(acquired)); if (secondThreadAcquiredLock.get()) { - Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); - secondThreadSavedWidget.set(updated != null && updated > 0); + SystemPatchApplier.WidgetTypeStats stats = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetTypes"); + secondThreadSavedWidget.set(stats != null && stats.updated() > 0); ReflectionTestUtils.invokeMethod(reconciler, "releaseAdvisoryLock"); } @@ -560,6 +626,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded"); @@ -607,6 +674,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); ReflectionTestUtils.invokeMethod(reconciler, "applyPatchIfNeeded"); @@ -834,6 +902,7 @@ public class SystemPatchApplierTest { Path widgetTypesDir = tempDir.resolve("widget_types"); Files.createDirectories(widgetTypesDir); when(installScripts.getWidgetTypesDir()).thenReturn(widgetTypesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("widget_bundles_missing")); when(imageService.getAllImageKeysByTenantId(TenantId.SYS_TENANT_ID)).thenReturn(Collections.emptySet()); @@ -854,4 +923,174 @@ public class SystemPatchApplierTest { verify(imageService, never()).createOrUpdateSystemImage(anyString(), any(byte[].class)); } + // --- updateWidgetBundles tests --- + + @Test + void whenWidgetBundlesDirDoesNotExist_thenReturnsZero() { + when(installScripts.getWidgetBundlesDir()).thenReturn(tempDir.resolve("missing_bundles")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleNotInDb_thenSkipWithoutCreation() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\"]}"); + + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(null); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleExistsAndHasNewFqn_thenMergeFqns() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\",\"new_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart", "bar_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(1, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService).updateWidgetsBundleWidgetFqns( + eq(TenantId.SYS_TENANT_ID), + eq(existingBundle.getId()), + argThat(fqns -> fqns.size() == 3 + && fqns.get(0).equals("line_chart") + && fqns.get(1).equals("bar_chart") + && fqns.get(2).equals("new_chart")) + ); + } + + @Test + void whenBundleExistsAndAllFqnsAlreadyLinked_thenNoLinkUpdate() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypeFqns\":[\"line_chart\",\"bar_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart", "bar_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleMetadataChanged_thenUpdateBundle() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"New Title\",\"description\":\"new\",\"order\":20}," + + "\"widgetTypeFqns\":[\"line_chart\"]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Old Title"); + existingBundle.setDescription("old"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of("line_chart")); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(1, updated); + verify(widgetsBundleService).saveWidgetsBundle(argThat(b -> + "New Title".equals(b.getTitle()) && "new".equals(b.getDescription()) && b.getOrder() == 20 + )); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + + @Test + void whenBundleAliasIsBlank_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("broken.json"), + "{\"widgetsBundle\":{\"alias\":\"\",\"title\":\"Broken\"}}"); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + @Test + void whenBundleJsonMissingWidgetsBundleField_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("broken.json"), "{\"foo\":\"bar\"}"); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + @Test + void whenBundleHasInlineWidgetTypes_thenThrowException() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10}," + + "\"widgetTypes\":[" + + "{\"fqn\":\"inline_chart\",\"name\":\"Inline\",\"descriptor\":{\"type\":\"latest\"}}" + + "]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + + assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles")); + verify(widgetTypeService, never()).saveWidgetType(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + } + + private WidgetsBundle createTestBundle(String alias, String title) { + WidgetsBundle bundle = new WidgetsBundle(); + bundle.setId(new WidgetsBundleId(UUID.randomUUID())); + bundle.setAlias(alias); + bundle.setTitle(title); + bundle.setTenantId(TenantId.SYS_TENANT_ID); + return bundle; + } + } From ee541633527b6146d7145009ad5e71c9e9c1b962 Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Tue, 5 May 2026 15:05:57 +0200 Subject: [PATCH 08/26] Skipped image field when comparing widgets bundle metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ImageService.replaceBase64WithImageUrl rewrites base64 data URIs into system-image URLs at save time. After install, the DB image is a 'tb-image;...' URL while the JSON file still carries a base64 data URI — naive string compare always reported a diff and caused every system bundle to be re-saved on every patch run. --- .../service/system/SystemPatchApplier.java | 5 +++- .../server/system/SystemPatchApplierTest.java | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java index e5bc65a047..a256888107 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java +++ b/application/src/main/java/org/thingsboard/server/service/system/SystemPatchApplier.java @@ -316,9 +316,12 @@ public class SystemPatchApplier { } private boolean isWidgetsBundleChanged(WidgetsBundle existing, WidgetsBundle file) { + // Image is intentionally NOT compared: the file always carries a base64 data URI, while the DB stores + // the system-image URL produced by ImageService.replaceBase64WithImageUrl on save. A naive string compare + // would always report a diff and re-save every system bundle on every patch run. Image content changes + // are out of scope for the patch applier — full reinstall covers them. return !Objects.equals(existing.getTitle(), file.getTitle()) || !Objects.equals(existing.getDescription(), file.getDescription()) - || !Objects.equals(existing.getImage(), file.getImage()) || !Objects.equals(existing.getOrder(), file.getOrder()) || existing.isScada() != file.isScada(); } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java index 4275b27560..f2485dc7d5 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemPatchApplierTest.java @@ -1010,6 +1010,33 @@ public class SystemPatchApplierTest { verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); } + @Test + void whenOnlyBundleImageFormatDiffers_thenNoUpdate() throws Exception { + Path bundlesDir = tempDir.resolve("widget_bundles"); + Files.createDirectories(bundlesDir); + when(installScripts.getWidgetBundlesDir()).thenReturn(bundlesDir); + + // File carries a base64 data URI; DB has the resolved system-image URL — same content, different format. + Files.writeString(bundlesDir.resolve("charts.json"), + "{\"widgetsBundle\":{\"alias\":\"charts\",\"title\":\"Charts\",\"description\":\"d\",\"order\":10," + + "\"image\":\"data:image/png;base64,iVBORw0KGgo\"}," + + "\"widgetTypeFqns\":[]}"); + + WidgetsBundle existingBundle = createTestBundle("charts", "Charts"); + existingBundle.setDescription("d"); + existingBundle.setOrder(10); + existingBundle.setImage("tb-image;/api/images/system/charts.png"); + when(widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, "charts")).thenReturn(existingBundle); + when(widgetTypeService.findWidgetFqnsByWidgetsBundleId(TenantId.SYS_TENANT_ID, existingBundle.getId())) + .thenReturn(List.of()); + + Integer updated = ReflectionTestUtils.invokeMethod(reconciler, "updateWidgetBundles"); + + assertEquals(0, updated); + verify(widgetsBundleService, never()).saveWidgetsBundle(any()); + verify(widgetTypeService, never()).updateWidgetsBundleWidgetFqns(any(), any(), any()); + } + @Test void whenBundleMetadataChanged_thenUpdateBundle() throws Exception { Path bundlesDir = tempDir.resolve("widget_bundles"); From 1a3dc833666c380100482330b055a083086e39ac Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Tue, 5 May 2026 16:42:48 +0300 Subject: [PATCH 09/26] Map widget: fix data aggregation for additional data keys and import/export widget JSON for polylines layer --- .../widget/maps/map-model.definition.ts | 119 ++++++------------ .../shared/models/widget/maps/map.models.ts | 2 + 2 files changed, 42 insertions(+), 79 deletions(-) diff --git a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts index 38cc0b52f4..681722c0c3 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts @@ -27,8 +27,10 @@ import { import { additionalMapDataSourcesToDatasources, BaseMapSettings, + latestMapDataLayerTypes, MapDataLayerSettings, MapDataLayerType, + mapDataLayerTypes, MapDataSourceSettings, mapDataSourceSettingsToDatasource, MapType @@ -46,30 +48,21 @@ interface MapDataLayerDsInfo extends AliasFilterPair { type ExportDataSourceInfo = {[dataLayerIndex: number]: MapDataLayerDsInfo}; -interface MapDatasourcesInfo { - trips?: ExportDataSourceInfo; - markers?: ExportDataSourceInfo; - polygons?: ExportDataSourceInfo; - circles?: ExportDataSourceInfo; +type MapDatasourcesInfo = { + [K in MapDataLayerType]?: ExportDataSourceInfo; +} & { additionalDataSources?: ExportDataSourceInfo; -} +}; export const MapModelDefinition: WidgetModelDefinition = { testWidget(widget: Widget): boolean { if (widget?.config?.settings) { const settings = widget.config.settings; if (settings.mapType && [MapType.image, MapType.geoMap].includes(settings.mapType)) { - if (settings.trips && Array.isArray(settings.trips)) { - return true; - } - if (settings.markers && Array.isArray(settings.markers)) { - return true; - } - if (settings.polygons && Array.isArray(settings.polygons)) { - return true; - } - if (settings.circles && Array.isArray(settings.circles)) { - return true; + for (const layerType of mapDataLayerTypes) { + if (Array.isArray(settings[layerType])) { + return true; + } } } } @@ -78,17 +71,11 @@ export const MapModelDefinition: WidgetModelDefinition = { prepareExportInfo(dashboard: Dashboard, widget: Widget): MapDatasourcesInfo { const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; const info: MapDatasourcesInfo = {}; - if (settings.trips?.length) { - info.trips = prepareExportDataSourcesInfo(dashboard, settings.trips); - } - if (settings.markers?.length) { - info.markers = prepareExportDataSourcesInfo(dashboard, settings.markers); - } - if (settings.polygons?.length) { - info.polygons = prepareExportDataSourcesInfo(dashboard, settings.polygons); - } - if (settings.circles?.length) { - info.circles = prepareExportDataSourcesInfo(dashboard, settings.circles); + for (const layerType of mapDataLayerTypes) { + const dataLayerSettings = settings[layerType]; + if (dataLayerSettings?.length) { + info[layerType] = prepareExportDataSourcesInfo(dashboard, dataLayerSettings); + } } if (settings.additionalDataSources?.length) { info.additionalDataSources = prepareExportDataSourcesInfo(dashboard, settings.additionalDataSources); @@ -96,59 +83,36 @@ export const MapModelDefinition: WidgetModelDefinition = { return info; }, updateFromExportInfo(widget: Widget, entityAliases: EntityAliases, filters: Filters, info: MapDatasourcesInfo): void { - const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - if (info?.trips) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.trips, info.trips); - } - if (info?.markers) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.markers, info.markers); - } - if (info?.polygons) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.polygons, info.polygons); - } - if (info?.circles) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.circles, info.circles); - } - if (info?.additionalDataSources) { - updateMapDatasourceFromExportInfo(entityAliases, filters, settings.additionalDataSources, info.additionalDataSources); + if (info && Object.keys(info).length) { + const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; + for (const layerType of mapDataLayerTypes) { + const layerInfo = info[layerType]; + const dataLayerSettings = settings[layerType]; + if (layerInfo && dataLayerSettings) { + updateMapDatasourceFromExportInfo(entityAliases, filters, dataLayerSettings, layerInfo); + } + } + if (info.additionalDataSources) { + updateMapDatasourceFromExportInfo(entityAliases, filters, settings.additionalDataSources, info.additionalDataSources); + } } }, datasources(widget: Widget): Datasource[] { - const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - const datasources: Datasource[] = []; - if (settings.trips?.length) { - datasources.push(...getMapDataLayersDatasources(settings.trips)); - } - if (settings.markers?.length) { - datasources.push(...getMapDataLayersDatasources(settings.markers)); - } - if (settings.polygons?.length) { - datasources.push(...getMapDataLayersDatasources(settings.polygons)); - } - if (settings.circles?.length) { - datasources.push(...getMapDataLayersDatasources(settings.circles)); - } - if (settings.additionalDataSources?.length) { - datasources.push(...additionalMapDataSourcesToDatasources(settings.additionalDataSources)); - } - return datasources; + return getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, mapDataLayerTypes); }, hasTimewindow(widget: Widget): boolean { const settings: BaseMapSettings = widget.config.settings as BaseMapSettings; - if (settings.trips?.length) { + const timeSeriesDataLayerTypes = mapDataLayerTypes.filter(t => !latestMapDataLayerTypes.includes(t)); + if (timeSeriesDataLayerTypes.some(layerType => settings[layerType]?.length)) { return true; - } else { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(settings); - return datasourcesHasAggregation(datasources); } + return datasourcesHasAggregation(getMapDataLayersDatasources(settings, latestMapDataLayerTypes, true)); }, datasourcesHasAggregation(widget: Widget): boolean { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(widget.config.settings as BaseMapSettings); - return datasourcesHasAggregation(datasources); + return datasourcesHasAggregation(getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, latestMapDataLayerTypes, true)); }, datasourcesHasOnlyComparisonAggregation(widget: Widget): boolean { - const datasources: Datasource[] = getMapLatestDataLayersDatasources(widget.config.settings as BaseMapSettings); - return datasourcesHasOnlyComparisonAggregation(datasources); + return datasourcesHasOnlyComparisonAggregation(getMapDataLayersDatasources(widget.config.settings as BaseMapSettings, latestMapDataLayerTypes, true)); } }; @@ -236,7 +200,7 @@ const prepareAliasAndFilterPair = (dashboard: Dashboard, settings: MapDataSource } } -const getMapDataLayersDatasources = (settings: MapDataLayerSettings[], +const getMapDataLayerDatasources = (settings: MapDataLayerSettings[], includeDataKeys = false, dataLayerType?: MapDataLayerType): Datasource[] => { const datasources: Datasource[] = []; settings.forEach((dsSettings) => { @@ -255,16 +219,13 @@ const getMapDataLayersDatasources = (settings: MapDataLayerSettings[], return datasources; }; -const getMapLatestDataLayersDatasources = (settings: BaseMapSettings): Datasource[] => { +const getMapDataLayersDatasources = (settings: BaseMapSettings, layerTypes: MapDataLayerType[], includeDataKeys = false): Datasource[] => { const datasources: Datasource[] = []; - if (settings.markers?.length) { - datasources.push(...getMapDataLayersDatasources(settings.markers, true, 'markers')); - } - if (settings.polygons?.length) { - datasources.push(...getMapDataLayersDatasources(settings.polygons, true, 'polygons')); - } - if (settings.circles?.length) { - datasources.push(...getMapDataLayersDatasources(settings.circles, true, 'circles')); + for (const layerType of layerTypes) { + const dataLayerSettings = settings[layerType]; + if (dataLayerSettings?.length) { + datasources.push(...getMapDataLayerDatasources(dataLayerSettings, includeDataKeys, layerType)); + } } if (settings.additionalDataSources?.length) { datasources.push(...additionalMapDataSourcesToDatasources(settings.additionalDataSources)); diff --git a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts index 0f1c979425..c846bd0522 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts @@ -204,6 +204,8 @@ export type MapDataLayerType = 'trips' | 'markers' | 'polygons' | 'circles' | 'p export const mapDataLayerTypes: MapDataLayerType[] = ['trips', 'markers', 'polygons', 'circles', 'polylines']; +export const latestMapDataLayerTypes: MapDataLayerType[] = ['markers', 'polygons', 'circles', 'polylines']; + export const mapDataLayerValid = (dataLayer: MapDataLayerSettings, type: MapDataLayerType): boolean => { if (!dataLayer.dsType || ![DatasourceType.function, DatasourceType.device, DatasourceType.entity].includes(dataLayer.dsType)) { return false; From 0c2ab1276b67274d409aea9d854f68515d433746 Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Tue, 5 May 2026 17:12:26 +0300 Subject: [PATCH 10/26] Map widget: derive mapDataLayerTypes from list of supported map data layers --- .../src/app/shared/models/widget/maps/map-model.definition.ts | 3 ++- ui-ngx/src/app/shared/models/widget/maps/map.models.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts index 681722c0c3..94eced7fcf 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map-model.definition.ts @@ -219,7 +219,8 @@ const getMapDataLayerDatasources = (settings: MapDataLayerSettings[], return datasources; }; -const getMapDataLayersDatasources = (settings: BaseMapSettings, layerTypes: MapDataLayerType[], includeDataKeys = false): Datasource[] => { +const getMapDataLayersDatasources = (settings: BaseMapSettings, + layerTypes: readonly MapDataLayerType[], includeDataKeys = false): Datasource[] => { const datasources: Datasource[] = []; for (const layerType of layerTypes) { const dataLayerSettings = settings[layerType]; diff --git a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts index c846bd0522..edfe03cc05 100644 --- a/ui-ngx/src/app/shared/models/widget/maps/map.models.ts +++ b/ui-ngx/src/app/shared/models/widget/maps/map.models.ts @@ -200,9 +200,9 @@ export const defaultBaseDataLayerSettings = (mapType: MapType): Partial Date: Wed, 6 May 2026 11:16:54 +0300 Subject: [PATCH 11/26] fixed web-ui and js-executor exe files for Windows --- msa/js-executor/package.json | 2 +- msa/web-ui/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index b096a3ae99..4dc5c85244 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -6,7 +6,7 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --out-path ./target ./target/src && node install.js", + "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --no-bytecode --public-packages \"*\" --public --out-path ./target ./target/src && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon --watch '.' --ext 'ts' --exec 'ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'NODE_ENV=production ts-node server.ts'", diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index c51aa6ed05..3040453ff5 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -6,7 +6,7 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --out-path ./target ./target/src && node install.js", + "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --no-bytecode --public-packages \"*\" --public --out-path ./target ./target/src && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web NODE_ENV=production ts-node server.ts'", From af659f36c055cdaaeba326b251ee9687608710d4 Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Tue, 5 May 2026 23:13:40 +0300 Subject: [PATCH 12/26] fix(csv): assign result of replace in splitCSV else branch The replace() call in the unquoted-field path was discarding its return value, leaving "" sequences unescaped instead of collapsing them to ". --- ui-ngx/src/app/shared/import-export/import-export.models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/import-export/import-export.models.ts b/ui-ngx/src/app/shared/import-export/import-export.models.ts index 2476922bf9..ab9def1697 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.models.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.models.ts @@ -210,7 +210,7 @@ function splitCSV(str: string, sep: string): string[] { foo = foo.shift().split(sep).concat(foo); } } else { - foo[x].replace(/""/g, '"'); + foo[x] = foo[x].replace(/""/g, '"'); } } return foo; From 739a33b72128fdca2a0da1a88a858cbad6bb5f2b Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Sun, 5 Apr 2026 21:49:58 +0200 Subject: [PATCH 13/26] Update tbel to 1.2.10 and add sandbox security tests Bump tbel dependency to 1.2.10 which blocks dangerous java.util subpackages (logging, zip, jar, prefs, spi) in TBEL sandbox. Add integration tests verifying sandbox blocks SocketHandler, ZipFile, FileHandler, JarFile, Preferences, and LocaleServiceProvider. --- .../service/script/TbelInvokeServiceTest.java | 84 +++++++++++++++++++ pom.xml | 2 +- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java index 54e1a39769..8618e515c8 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeServiceTest.java @@ -217,6 +217,90 @@ class TbelInvokeServiceTest extends AbstractTbelInvokeTest { assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull(); } + @Test + void givenForbiddenSocketHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.logging.SocketHandler(\"127.0.0.1\", 9999)"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.SocketHandler"); + }); + } + + @Test + void givenForbiddenZipFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.zip.ZipFile(\"/tmp/test.zip\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.zip.ZipFile"); + }); + } + + @Test + void givenForbiddenFileHandler_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.logging.FileHandler(\"/tmp/test.log\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.logging.FileHandler"); + }); + } + + @Test + void givenForbiddenJarFile_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.jar.JarFile(\"/tmp/test.jar\")"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.jar.JarFile"); + }); + } + + @Test + void givenForbiddenPreferences_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("java.util.prefs.Preferences.userRoot()"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getMessage()).contains("unresolvable property or identifier: java"); + }); + } + + @Test + void givenForbiddenLocaleServiceProvider_whenInvoking_thenThrowsRuntimeError() throws ExecutionException, InterruptedException { + UUID scriptId = evalScript("new java.util.spi.LocaleServiceProvider()"); + assertThatThrownBy(() -> invokeScript(scriptId, "{\"temperature\":25}")) + .isInstanceOf(ExecutionException.class) + .cause() + .isInstanceOf(TbScriptException.class) + .asInstanceOf(type(TbScriptException.class)) + .satisfies(ex -> { + assertThat(ex.getErrorCode()).isEqualTo(TbScriptException.ErrorCode.RUNTIME); + assertThat(ex.getCause().getMessage()).contains("could not resolve class: java.util.spi.LocaleServiceProvider"); + }); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScriptResultString(scriptId, "{}"); diff --git a/pom.xml b/pom.xml index 328c4782b3..c1725a3952 100755 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ 3.9.5 3.25.5 1.76.0 - 1.2.9 + 1.2.10 1.18.46 1.2.5 1.2.5 From 2395cd8c4ed5ecba58aa0e5cfd41743e2e1636a9 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 6 May 2026 15:37:26 +0300 Subject: [PATCH 14/26] split pkg per target to keep bytecode on Linux --no-bytecode is only required for the Windows target; running pkg as a single multi-target invocation applied it to Linux too, dropping V8 bytecode from the Linux exe. Split into two pkg invocations so Linux keeps bytecode and only the Windows build relaxes it. --- msa/js-executor/package.json | 2 +- msa/web-ui/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 4dc5c85244..1f1fee52d5 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -6,7 +6,7 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --no-bytecode --public-packages \"*\" --public --out-path ./target ./target/src && node install.js", + "pkg": "tsc && pkg -t node22-linux-x64 --output ./target/thingsboard-js-executor-linux ./target/src && pkg -t node22-win-x64 --no-bytecode --public-packages \"*\" --public --output ./target/thingsboard-js-executor-win.exe ./target/src && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon --watch '.' --ext 'ts' --exec 'ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'NODE_ENV=production ts-node server.ts'", diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 3040453ff5..aaac7f8b67 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -6,7 +6,7 @@ "main": "server.ts", "bin": "server.js", "scripts": { - "pkg": "tsc && pkg -t node22-linux-x64,node22-win-x64 --no-bytecode --public-packages \"*\" --public --out-path ./target ./target/src && node install.js", + "pkg": "tsc && pkg -t node22-linux-x64 --output ./target/thingsboard-web-ui-linux ./target/src && pkg -t node22-win-x64 --no-bytecode --public-packages \"*\" --public --output ./target/thingsboard-web-ui-win.exe ./target/src && node install.js", "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'WEB_FOLDER=./target/web NODE_ENV=production ts-node server.ts'", From d7557339c21be678d93f0efe663c5117ee75df0b Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 6 May 2026 16:12:15 +0300 Subject: [PATCH 15/26] Added lz4 as compression option into yml files --- application/src/main/resources/thingsboard.yml | 4 ++-- edqs/src/main/resources/edqs.yml | 4 ++-- msa/js-executor/config/custom-environment-variables.yml | 2 +- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 2 +- transport/coap/src/main/resources/tb-coap-transport.yml | 4 ++-- transport/http/src/main/resources/tb-http-transport.yml | 2 +- transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 4 ++-- transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 4 ++-- transport/snmp/src/main/resources/tb-snmp-transport.yml | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 84bc3f7afc..2c24caa1f2 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1689,8 +1689,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 010994dc88..4dc404e2ca 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -103,8 +103,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 6a98361c72..9923fe4ed3 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -34,7 +34,7 @@ kafka: partitions_consumed_concurrently: "TB_KAFKA_PARTITIONS_CONSUMED_CONCURRENTLY" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS" connectionTimeout: "TB_KAFKA_CONNECTION_TIMEOUT_MS" - compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip or uncompressed + compression: "TB_QUEUE_KAFKA_COMPRESSION" # gzip, lz4 or none topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" use_confluent_cloud: "TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD" client_id: "KAFKA_CLIENT_ID" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 8314985622..9ed3299606 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,7 +73,7 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 3c1ef94f09..1554b3fc24 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -285,8 +285,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 1b221d1fd9..c878887a01 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -235,7 +235,7 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 6140122062..51b1ad0a2b 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -335,8 +335,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index ac02fa396b..dfe35db29c 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -268,8 +268,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 656c01524d..d021030a91 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -207,8 +207,8 @@ queue: acks: "${TB_KAFKA_ACKS:all}" # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" - # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip - compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values: none, gzip or lz4 + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none, gzip or lz4 # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record From 183def96f36d97cd2f883492223f1f539f2afd5f Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Wed, 6 May 2026 15:24:08 +0200 Subject: [PATCH 16/26] Fixed CVE-2026-40682, CVE-2026-42027 --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index c1725a3952..5deaf5e427 100755 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,7 @@ 1.7.5 3.8.0 1.8.0-TB + 2.5.9 2.38.0 1.24 1.11.0 @@ -1347,6 +1348,11 @@ postgresql ${postgresql.version} + + org.apache.opennlp + opennlp-tools + ${opennlp-tools.version} + commons-io commons-io From d3e526058f4d6e25fbca9d450be0d01f96bb2c91 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 6 May 2026 16:51:04 +0300 Subject: [PATCH 17/26] Code review changes --- .../server/transport/snmp/service/SnmpTransportService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 5105b35d1d..a15a21b6e3 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -390,9 +390,11 @@ public class SnmpTransportService implements TbTransportService, CommandResponde JsonObject responseData = responseDataMappers.get(requestContext.getCommunicationSpec()).map(response, requestContext); if (responseData.size() == 0) { - log.warn("[{}] No values in the response for spec {}. Response PDUs: {}, Mappings count: {}", + log.warn("[{}] No values in the response for spec {}. Response PDUs count: {}, Mappings count: {}", sessionContext.getDeviceId(), requestContext.getCommunicationSpec(), - response, requestContext.getResponseMappings() != null ? requestContext.getResponseMappings().size() : 0); + response.size(), requestContext.getResponseMappings().size()); + log.debug("[{}] No values in the response for spec {}. Response PDUs: {}", + sessionContext.getDeviceId(), requestContext.getCommunicationSpec(), response); throw new IllegalArgumentException("No values in the response"); } From 947e75a56adf1edbed79d86ca0eae5fececa1751 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 7 May 2026 13:33:46 +0300 Subject: [PATCH 18/26] Code review changes --- msa/js-executor/package.json | 5 +++-- msa/js-executor/queue/kafkaTemplate.ts | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index dfa50f6765..7cbd99ea48 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -13,11 +13,11 @@ "build": "tsc" }, "dependencies": { + "@2l/kafkajs-lz4": "^1.3.2", "config": "^4.1.1", "express": "^5.1.0", "js-yaml": "^4.1.1", "kafkajs": "^2.2.4", - "@2l/kafkajs-lz4": "^1.3.2", "long": "^5.3.2", "uuid-parse": "^1.1.0", "winston": "^3.17.0", @@ -47,7 +47,8 @@ }, "pkg": { "assets": [ - "node_modules/config/**/*.*" + "node_modules/config/**/*.*", + "node_modules/@antoniomuso/lz4-napi-*/**/*.node" ] } } diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 5a340b2cfd..1114190e64 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -31,14 +31,11 @@ import { Producer, TopicMessages } from 'kafkajs'; -import LZ4Codec from '@2l/kafkajs-lz4'; import { isNotEmptyStr } from '../api/utils'; import { KeyObject } from 'tls'; import process, { exit, kill } from 'process'; -CompressionCodecs[CompressionTypes.LZ4] = new LZ4Codec().codec; - export class KafkaTemplate implements IQueue { private logger = _logger(`kafkaTemplate`); @@ -50,15 +47,25 @@ export class KafkaTemplate implements IQueue { private linger = Number(config.get('kafka.linger_ms')); private requestTimeout = Number(config.get('kafka.requestTimeout')); private connectionTimeout = Number(config.get('kafka.connectionTimeout')); - private compressionType = KafkaTemplate.resolveCompressionType(config.get('kafka.compression')); + private compressionType = this.resolveCompressionType(config.get('kafka.compression')); - private static resolveCompressionType(compression: string): CompressionTypes { + private resolveCompressionType(compression: string): CompressionTypes { switch (compression) { case 'gzip': return CompressionTypes.GZIP; - case 'lz4': + case 'lz4': { + // Load the LZ4 codec lazily so users who don't enable LZ4 don't take a hard + // dependency on the lz4-napi native binary (e.g. inside pkg-built executables). + const LZ4Codec = require('@2l/kafkajs-lz4').default; + CompressionCodecs[CompressionTypes.LZ4] = new LZ4Codec().codec; return CompressionTypes.LZ4; + } + case 'none': + return CompressionTypes.None; default: + if (isNotEmptyStr(compression)) { + this.logger.warn('Unknown kafka.compression value "%s"; falling back to no compression. Supported values: gzip, lz4, none.', compression); + } return CompressionTypes.None; } } From 9a84ea35a0eaa58ec233a1d862df86416c3e0ed4 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 7 May 2026 14:48:34 +0300 Subject: [PATCH 19/26] Bump io.netty:netty-bom from 4.1.132.Final to 4.1.133.Final to fix CVE-2026-42579, CVE-2026-42583, CVE-2026-42584, and CVE-2026-42587 --- pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pom.xml b/pom.xml index 5deaf5e427..24f8f9e1c2 100755 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,7 @@ 3.5.13 3.18.0 42.7.11 + 4.1.133.Final 2.4.0-b180830.0359 0.12.5 0.10 @@ -1006,6 +1007,15 @@ + + + io.netty + netty-bom + ${netty.version} + pom + import + + org.springframework.boot spring-boot-dependencies From 9dcbb3d9f93032e6fff21ac69916605d7c72c9b2 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 7 May 2026 15:24:14 +0300 Subject: [PATCH 20/26] Fixed startup of js-executor for lz4 compression --- msa/js-executor/queue/kafkaTemplate.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 1114190e64..175abd80f4 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -56,7 +56,10 @@ export class KafkaTemplate implements IQueue { case 'lz4': { // Load the LZ4 codec lazily so users who don't enable LZ4 don't take a hard // dependency on the lz4-napi native binary (e.g. inside pkg-built executables). - const LZ4Codec = require('@2l/kafkajs-lz4').default; + // The package re-assigns module.exports = LZ4Codec, which wipes the __esModule + // flag and the .default property — so accept either shape. + const lz4Module = require('@2l/kafkajs-lz4'); + const LZ4Codec = lz4Module.default || lz4Module; CompressionCodecs[CompressionTypes.LZ4] = new LZ4Codec().codec; return CompressionTypes.LZ4; } From e5734208e95e748b3dff2a56238bde244171f8a6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 8 May 2026 12:08:45 +0200 Subject: [PATCH 21/26] Hardened tb-js-executor sandbox script invocation (JVN#16937365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The args array passed into the sandbox carried the host realm prototype chain, so a script could reach the host Function constructor via args.constructor.constructor and execute arbitrary code in the host process (read files, run shell commands, dump env vars). Construct args inside the sandbox context using vm.runInContext('[]'), then populate with string primitives. The resulting array's prototype chain belongs to the sandbox realm, so constructor traversal cannot escape. Strings are primitives and safe to cross the realm boundary. Affects use_sandbox=true path only. The use_sandbox=false path (invokeFunction) is intentionally left as-is and explicitly marked as dangerous-by-design — it compiles and runs user-supplied scripts in the host realm via vm.compileFunction (parsingContext only isolates parsing, not execution). It remains as a documented performance trade-off for trusted, non-public clusters; a startup WARN is logged when script.use_sandbox=false, and an operator-facing yaml comment sits next to the setting in config/default.yml. Reported by Hiroki Imai, LAC Co., Ltd. --- msa/js-executor/api/jsExecutor.ts | 25 ++++++++++++++++++++++--- msa/js-executor/config/default.yml | 6 ++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/msa/js-executor/api/jsExecutor.ts b/msa/js-executor/api/jsExecutor.ts index 855ad99932..ab916a2760 100644 --- a/msa/js-executor/api/jsExecutor.ts +++ b/msa/js-executor/api/jsExecutor.ts @@ -15,14 +15,22 @@ /// import vm, { Script } from 'vm'; +import { _logger } from '../config/logger'; export type TbScript = Script | Function; export class JsExecutor { useSandbox: boolean; + private logger = _logger('JsExecutor'); constructor(useSandbox: boolean) { this.useSandbox = useSandbox; + if (!useSandbox) { + this.logger.warn( + 'script.use_sandbox=false: dangerous by design — user-supplied scripts run in the host realm with no isolation. ' + + 'Use only as a performance trade-off in trusted, non-public clusters.' + ); + } } compileScript(code: string): Promise { @@ -56,9 +64,15 @@ export class JsExecutor { private invokeScript(script: Script, args: string[], timeout: number | undefined): Promise { return new Promise((resolve, reject) => { try { - const sandbox = Object.create(null); - sandbox.args = args; - const result = script.runInNewContext(sandbox, {timeout: timeout}); + const sandbox = vm.createContext(Object.create(null)); + // Construct args inside the sandbox context so it inherits sandbox-realm + // prototypes; prevents prototype-based escapes from the host realm. + const ctxArgs = vm.runInContext('[]', sandbox) as string[]; + for (let i = 0; i < args.length; i++) { + ctxArgs[i] = String(args[i]); + } + sandbox.args = ctxArgs; + const result = script.runInContext(sandbox, {timeout: timeout}); resolve(result); } catch (err) { reject(err); @@ -67,6 +81,11 @@ export class JsExecutor { } + // DANGEROUS BY DESIGN: the non-sandbox path. vm.compileFunction's + // parsingContext only isolates *parsing*, not *execution* — the resulting + // function runs in the host realm with full access to host globals + // (process, require, etc.). Enabled only via script.use_sandbox=false as + // a performance trade-off in trusted clusters. private createFunction(code: string): Promise { return new Promise((resolve, reject) => { try { diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 5939b0e29a..9a33190069 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -50,6 +50,12 @@ logger: filename: "tb-js-executor-%DATE%.log" script: + # WARNING: setting this to "false" is DANGEROUS BY DESIGN. The non-sandbox + # path compiles and runs user-supplied scripts in the host realm via + # vm.compileFunction; it provides no isolation and exposes the host process + # (file system, environment variables, child_process, etc.) to script + # authors. Use "false" only as a performance trade-off in trusted, + # non-public clusters where every script author is fully trusted. use_sandbox: "true" memory_usage_trace_frequency: "1000" script_body_trace_frequency: "10000" From e329c8d162b52dd29618bbaae8e0ed92338584ce Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 8 May 2026 12:08:56 +0200 Subject: [PATCH 22/26] Added Node unit tests for tb-js-executor sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four test cases under describe('js-executor'): - sandbox isolates args from host realm (JVN#16937365 — regression guard) - sandbox passes string args through unchanged - non-sandbox path does not isolate from host realm (documented contract) - non-sandbox path passes string args through unchanged Tests use Node's built-in node:test + node:assert (zero new devDependencies; ts-node was already there). Two npm scripts: test — spec output for local dev test:ci — spec to stdout + Node's built-in junit reporter to target/surefire-reports/TEST-js-executor.xml Wired 'yarn test:ci' into the Maven 'test' phase via frontend-maven-plugin, so 'mvn test -pl=msa/js-executor' produces JUnit XML that TeamCity's Maven runner auto-discovers under the 'js-executor' suite name. TEST_FAST.md picks up the same step. tsconfig excludes test/ from the production pkg bundle. --- TEST_FAST.md | 1 + msa/js-executor/package.json | 2 +- msa/js-executor/pom.xml | 11 ++++ msa/js-executor/test/jsExecutor.test.ts | 83 +++++++++++++++++++++++++ msa/js-executor/tsconfig.json | 2 +- 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 msa/js-executor/test/jsExecutor.test.ts diff --git a/TEST_FAST.md b/TEST_FAST.md index eb2013c601..fbea72db9c 100644 --- a/TEST_FAST.md +++ b/TEST_FAST.md @@ -10,6 +10,7 @@ export SUREFIRE_JAVA_OPTS="-Xmx1200m -Xss256k -XX:+ExitOnOutOfMemoryError" mvn clean install -T6 -DskipTests -Dpkg.skip=true mvn test -pl='!application,!dao,!ui-ngx,!msa/js-executor,!msa/web-ui' -T4 +mvn test -pl='msa/js-executor' mvn test -pl dao -Dparallel=packages -DforkCount=4 mvn test -pl application -Dtest='!**/nosql/**,org.thingsboard.server.controller.**' -DforkCount=6 -Dparallel=classes -Dsurefire.rerunFailingTestsCount=2 -Dsurefire.failOnFlakeCount=5 diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 13b7c01d81..9cd29e35d5 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -7,7 +7,7 @@ "bin": "server.js", "scripts": { "pkg": "tsc && pkg -t node22-linux-x64 --output ./target/thingsboard-js-executor-linux ./target/src && pkg -t node22-win-x64 --no-bytecode --public-packages \"*\" --public --output ./target/thingsboard-js-executor-win.exe ./target/src && node install.js", - "test": "echo \"Error: no test specified\" && exit 1", + "test": "mkdir -p target/surefire-reports && node --require ts-node/register --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=target/surefire-reports/TEST-js-executor.xml test/jsExecutor.test.ts", "start": "nodemon --watch '.' --ext 'ts' --exec 'ts-node server.ts'", "start-prod": "nodemon --watch '.' --ext 'ts' --exec 'NODE_ENV=production ts-node server.ts'", "build": "tsc" diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index 58404c0152..2f7434b472 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -116,6 +116,17 @@ --mutex network run pkg + + yarn test + + yarn + + test + + ${maven.test.skip} + --mutex network run test + + diff --git a/msa/js-executor/test/jsExecutor.test.ts b/msa/js-executor/test/jsExecutor.test.ts new file mode 100644 index 0000000000..7777030ce9 --- /dev/null +++ b/msa/js-executor/test/jsExecutor.test.ts @@ -0,0 +1,83 @@ +/// +/// Copyright © 2016-2026 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { JsExecutor } from '../api/jsExecutor'; + +// describe('js-executor') groups all cases under +// in the JUnit XML so they show up under that suite in TeamCity's Tests tab, +// alongside thousands of Java tests. +describe('js-executor', () => { + +test('sandbox isolates args from host realm (JVN#16937365)', async () => { + const exec = new JsExecutor(true); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + var F = args.constructor.constructor; + var p = F("return process")(); + return p && p.mainModule ? 'reached-host' : 'isolated'; + }`); + await assert.rejects( + exec.executeScript(script, ['{}', '{}', 'POST_TELEMETRY_REQUEST'], 5000), + /process is not defined/, + 'host process must not be reachable from inside the sandbox', + ); +}); + +test('sandbox passes string args through unchanged', async () => { + const exec = new JsExecutor(true); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + return { msgIsString: typeof msg === 'string', count: args.length, first: args[0] }; + }`); + const out = await exec.executeScript(script, ['hello', '{}', 'X'], 5000); + // Field-by-field: the returned object is owned by the sandbox realm, so + // its prototype is not the host Object.prototype and deepStrictEqual would + // reject it on prototype mismatch even when the values match. + assert.equal(out.msgIsString, true); + assert.equal(out.count, 3); + assert.equal(out.first, 'hello'); +}); + +// The use_sandbox=false path is intentionally non-isolating: scripts compile +// and run in the host realm via vm.compileFunction. The two tests below codify +// that documented contract so any future behavior change shows up as a test +// failure and forces a deliberate update of the docs and threat model. + +test('non-sandbox path does not isolate from host realm (documented contract)', async () => { + const exec = new JsExecutor(false); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + // Non-destructive host-reach probe: typeof process.platform is 'string' + // only if the host process object is reachable. + var F = args.constructor.constructor; + return F('return typeof process.platform')(); + }`); + const out = await exec.executeScript(script, ['{}', '{}', 'X']); + assert.equal(out, 'string', + 'use_sandbox=false is documented as non-isolating; if this fails, the path was changed and docs/threat model must be updated'); +}); + +test('non-sandbox path passes string args through unchanged', async () => { + const exec = new JsExecutor(false); + const script = await exec.compileScript(`function(msg, metadata, msgType){ + return { msgIsString: typeof msg === 'string', count: args.length, first: args[0] }; + }`); + const out = await exec.executeScript(script, ['hello', '{}', 'X']); + assert.equal(out.msgIsString, true); + assert.equal(out.count, 3); + assert.equal(out.first, 'hello'); +}); + +}); // describe('js-executor') diff --git a/msa/js-executor/tsconfig.json b/msa/js-executor/tsconfig.json index b633ffc768..a1f7b25466 100644 --- a/msa/js-executor/tsconfig.json +++ b/msa/js-executor/tsconfig.json @@ -9,5 +9,5 @@ "skipLibCheck": true, "strictPropertyInitialization": false }, - "exclude": ["node_modules", "target"] + "exclude": ["node_modules", "target", "test"] } From 6d00e0432a2316750da2c3c3f639e7c944991f88 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 8 May 2026 12:09:08 +0200 Subject: [PATCH 23/26] Added regression tests for /api/ruleChain/testScript Two test layers covering the controller surface that the JVN PoC uses: Java unit (Spring MockMvc): RuleChainControllerTest#testScriptForbiddenForCustomer asserts a customer JWT against POST /api/ruleChain/testScript returns 403, locking in the existing @PreAuthorize('TENANT_ADMIN') guard. Black-box (live docker-compose): JsExecutorSandboxIsolationTest#testRuleChainScriptCannotReachHostProcess posts the JVN exploit payload as a tenant admin and asserts the response carries error='process is not defined'. End-to-end through tb-node -> Kafka -> tb-js-executor with use_sandbox=true. Registered the new org.thingsboard.server.msa.security package in the connectivity TestNG suite so the black-box runner picks it up. Added a thin TestRestClient.testRuleChainScript() helper. --- .../controller/RuleChainControllerTest.java | 17 +++++ .../server/msa/TestRestClient.java | 10 +++ .../JsExecutorSandboxIsolationTest.java | 73 +++++++++++++++++++ .../src/test/resources/connectivity.xml | 1 + 4 files changed, 101 insertions(+) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java index f8cd8e0f33..a485c02bb8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java @@ -405,4 +405,21 @@ public class RuleChainControllerTest extends AbstractControllerTest { return doPost("/api/ruleChain", ruleChain, RuleChain.class); } + @Test + public void testScriptForbiddenForCustomer() throws Exception { + loginCustomerUser(); + + doPost("/api/ruleChain/testScript", (Object) """ + { + "script": "return msg;", + "scriptType": "update", + "argNames": ["msg", "metadata", "msgType"], + "msg": "{}", + "metadata": {}, + "msgType": "POST_TELEMETRY_REQUEST" + } + """) + .andExpect(status().isForbidden()); + } + } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 7b2cfc68d9..1e14dbd70b 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -328,6 +328,16 @@ public class TestRestClient { .statusCode(HTTP_OK); } + public JsonNode testRuleChainScript(Object body) { + return given().spec(requestSpec) + .body(body) + .post("/api/ruleChain/testScript") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + private String getUrlParams(PageLink pageLink) { String urlParams = "pageSize={pageSize}&page={page}"; if (!isEmpty(pageLink.getTextSearch())) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java new file mode 100644 index 0000000000..1328382c5d --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/security/JsExecutorSandboxIsolationTest.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.security; + +import com.fasterxml.jackson.databind.JsonNode; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.thingsboard.server.msa.AbstractContainerTest; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JsExecutorSandboxIsolationTest extends AbstractContainerTest { + + @BeforeClass + public void beforeClass() { + testRestClient.login("tenant@thingsboard.org", "tenant"); + } + + @AfterClass + public void afterClass() { + testRestClient.resetToken(); + } + + /** + * Black-box regression for JVN#16937365: a tenant admin must not be able + * to escape the tb-js-executor sandbox via the host-realm prototype chain + * exposed through the script's `args` argument. Runs against the live + * docker-compose deployment, which uses script.use_sandbox=true and + * JS_EVALUATOR=remote (Kafka -> tb-js-executor). + */ + @Test + public void testRuleChainScriptCannotReachHostProcess() { + JsonNode response = testRestClient.testRuleChainScript(""" + { + "script": "var F = args.constructor.constructor; var p = F('return process')(); return { reachedHost: !!(p && p.mainModule) };", + "scriptType": "update", + "argNames": ["msg", "metadata", "msgType"], + "msg": "{}", + "metadata": {}, + "msgType": "POST_TELEMETRY_REQUEST" + } + """); + + // The sandboxed run must reject the escape attempt: the host `process` + // global is not defined inside the sandbox realm, so executing the + // synthesized function `F("return process")` throws. + assertThat(response.has("error")).isTrue(); + String error = response.get("error").asText(); + assertThat(error) + .as("sandbox must block host-realm reach via args.constructor.constructor; full error: %s", error) + .contains("process is not defined"); + + // Defense in depth: even if the script somehow returned, output must + // not indicate that the host process was reached. + if (response.hasNonNull("output")) { + assertThat(response.get("output").asText()).doesNotContain("\"reachedHost\":true"); + } + } +} diff --git a/msa/black-box-tests/src/test/resources/connectivity.xml b/msa/black-box-tests/src/test/resources/connectivity.xml index a1cbaf4af6..7cd96bb429 100644 --- a/msa/black-box-tests/src/test/resources/connectivity.xml +++ b/msa/black-box-tests/src/test/resources/connectivity.xml @@ -25,6 +25,7 @@ + \ No newline at end of file From 3aaa4d2fd28ffc498e478708800b82ea90c5f4fd Mon Sep 17 00:00:00 2001 From: Maksym Tsymbarov Date: Thu, 14 May 2026 12:00:09 +0200 Subject: [PATCH 24/26] Remove unnecessary DomSanitizer bypass in photo camera input widget --- .../widget/lib/photo-camera-input.component.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts index 060a74aebd..541b6ab384 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/photo-camera-input.component.ts @@ -24,7 +24,7 @@ import { ViewChild, ViewEncapsulation } from '@angular/core'; -import { DomSanitizer, SafeUrl } from '@angular/platform-browser'; + import { ImageService } from '@app/core/public-api'; import { AppState } from '@core/core.state'; import { AttributeService } from '@core/http/attribute.service'; @@ -63,8 +63,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On protected store: Store, private imageService: ImageService, private utils: UtilsService, - private attributeService: AttributeService, - private sanitizer: DomSanitizer + private attributeService: AttributeService ) { super(store); } @@ -115,8 +114,8 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On isLoading = false; singleDevice = true; updatePhoto = false; - previewPhoto: SafeUrl; - lastPhoto: SafeUrl; + previewPhoto: string; + lastPhoto: string; datasourceDetected = false; private mimeType: string; @@ -176,7 +175,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On private updateWidgetData(data: Array) { const keyData = data[0].data; if (keyData?.length && isString(keyData[0][1])) { - this.lastPhoto = keyData[0][1].startsWith('data:image/') ? this.sanitizer.bypassSecurityTrustUrl(keyData[0][1]) : keyData[0][1]; + this.lastPhoto = keyData[0][1]; } } @@ -309,7 +308,7 @@ export class PhotoCameraInputWidgetComponent extends PageComponent implements On const file = new File([blob], fileName, { type: this.mimeType }); return this.imageService.uploadImage(file, fileName); }), - map((imageInfo) => + map((imageInfo) => this.settings.usePublicGalleryLink ? imageInfo.publicLink : imageInfo.link ) ); From bd0427e8756be9c6acca78516e1f5b5a4b8200fd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 18 May 2026 10:46:36 +0300 Subject: [PATCH 25/26] Bump tomcat.version from 10.1.54 to 10.1.55 to fix CVE-2026-41284 and CVE-2026-43512 --- pom.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pom.xml b/pom.xml index 24f8f9e1c2..4a8958ffa3 100755 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,7 @@ 3.18.0 42.7.11 4.1.133.Final + 10.1.55 2.4.0-b180830.0359 0.12.5 0.10 @@ -1016,6 +1017,23 @@ import + + + org.apache.tomcat.embed + tomcat-embed-core + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-el + ${tomcat.version} + + + org.apache.tomcat.embed + tomcat-embed-websocket + ${tomcat.version} + + org.springframework.boot spring-boot-dependencies From 6079cbb1808a434cd2a757a87e6b981141a2874f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 19 May 2026 10:31:46 +0300 Subject: [PATCH 26/26] fixed openapi descriptions for arrays --- .../server/common/data/ai/dto/TbUserMessage.java | 10 ++++------ .../common/data/query/AvailableEntityKeys.java | 12 ++++++++---- .../common/data/query/AvailableEntityKeysV2.java | 14 ++++++-------- .../data/sync/ie/WidgetsBundleExportData.java | 3 ++- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java b/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java index e1f0b84a4d..9e91adb2c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ai/dto/TbUserMessage.java @@ -25,12 +25,10 @@ import java.util.List; public record TbUserMessage( @NotEmpty @Valid - @ArraySchema( - arraySchema = @Schema( - requiredMode = Schema.RequiredMode.REQUIRED, - description = "A list of content parts that make up the complete user prompt" - ), - schema = @Schema(ref = "#/components/schemas/TbContent") + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "A list of content parts that make up the complete user prompt" ) + @ArraySchema(schema = @Schema(ref = "#/components/schemas/TbContent")) List contents ) {} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java index a20d0dca55..532f7f2e1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java @@ -37,17 +37,21 @@ public record AvailableEntityKeys( ) Set entityTypes, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "List of unique time series key names available on the matched entities." + ) @ArraySchema( - arraySchema = @Schema(description = "List of unique time series key names available on the matched entities."), schema = @Schema(implementation = String.class, example = "temperature"), uniqueItems = true ) List timeseries, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @Schema( + requiredMode = Schema.RequiredMode.REQUIRED, + description = "List of unique attribute key names available on the matched entities." + ) @ArraySchema( - arraySchema = @Schema(description = "List of unique attribute key names available on the matched entities."), schema = @Schema(implementation = String.class, example = "serialNumber"), uniqueItems = true ) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java index 536b70e4ff..608177b9ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java @@ -41,15 +41,13 @@ public record AvailableEntityKeysV2( ) Set entityTypes, - @ArraySchema( - arraySchema = @Schema( - description = """ - List of unique time series keys available on the matched entities, sorted alphabetically. - Omitted when timeseries keys were not requested.""", - nullable = true - ), - schema = @Schema(implementation = KeyInfo.class) + @Schema( + description = """ + List of unique time series keys available on the matched entities, sorted alphabetically. + Omitted when timeseries keys were not requested.""", + nullable = true ) + @ArraySchema(schema = @Schema(implementation = KeyInfo.class)) @Nullable List timeseries, @Schema( diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java index 62140abdd1..6883fff6d4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java @@ -36,7 +36,8 @@ public class WidgetsBundleExportData extends EntityExportData { @Override public EntityType getEntityType() { return EntityType.WIDGETS_BUNDLE; } - @ArraySchema(arraySchema = @Schema(description = "List of widgets in the bundle"), schema = @Schema(implementation = JsonNode.class)) + @Schema(description = "List of widgets in the bundle") + @ArraySchema(schema = @Schema(implementation = JsonNode.class)) @JsonProperty(index = 3) private List widgets;