From 0468451f3ba02983e540e26f968305dd7a2ed04a Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 12 Jan 2026 10:52:29 +0200 Subject: [PATCH 01/71] added ws update on telemetry deletion --- .../DefaultTbLocalSubscriptionService.java | 4 +-- .../server/controller/WebsocketApiTest.java | 35 +++++++++++++++++++ .../server/common/data/kv/TsKvEntry.java | 5 +++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 51663eea9c..df28da2dad 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -348,7 +348,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (sub.isLatestValues()) { for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); - if (stateTs == null || kv.getTs() >= stateTs) { + if (stateTs == null || kv.getTs() >= stateTs || kv.isDeletedEntryMarker()) { if (updateData == null) { updateData = new ArrayList<>(); } @@ -362,7 +362,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); if (stateTs != null) { - if (!sub.isLatestValues() || kv.getTs() >= stateTs) { + if (!sub.isLatestValues() || kv.getTs() >= stateTs || kv.isDeletedEntryMarker()) { if (updateData == null) { updateData = new ArrayList<>(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 87ba0ec3e8..e88808ad2a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -726,6 +726,41 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertNull(msg); } + @Test + public void testShouldSendWsUpdateMessageWhenTelemetryWasDeleted() throws Exception { + long now = System.currentTimeMillis() - 100; + TsKvEntry dataPoint = new BasicTsKvEntry(now, new LongDataEntry("temperature", 42L)); + List tsData = List.of(dataPoint); + sendTelemetry(device, tsData); + + List keys = List.of(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataUpdate update = getWsClient().subscribeLatestUpdate(keys, dtf); + + Assert.assertEquals(1, update.getCmdId()); + PageData pageData = update.getData(); + Assert.assertNotNull(pageData); + Assert.assertEquals(1, pageData.getData().size()); + Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); + Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(now, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("42", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + + // delete telemetry + getWsClient().registerWaitForUpdate(); + doDeleteAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/delete?keys=temperature&deleteAllDataForKeys=true", String.class); + update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); + + Assert.assertEquals(1, update.getCmdId()); + + List listData = update.getUpdate(); + Assert.assertNotNull(listData); + Assert.assertEquals(1, listData.size()); + Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); + Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(new TsValue(0, ""), tsValue); + } + @Test public void testEntityDataLatestAttrWsCmd() throws Exception { long now = System.currentTimeMillis(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index cb4092f433..eca0609704 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -37,4 +37,9 @@ public interface TsKvEntry extends KvEntry, HasVersion { return new TsValue(getTs(), getValueAsString()); } + @JsonIgnore + default boolean isDeletedEntryMarker() { + return getTs() == 0 && (getValue() == null || getValueAsString().isEmpty()); + } + } From 97d68dda90433a441eac439bb1d377a9b58bd978 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 28 Jan 2026 12:00:25 +0200 Subject: [PATCH 02/71] Ota package unlink data object --- .../housekeeper/HousekeeperServiceTest.java | 10 ++-- .../AlarmsDeletionHousekeeperTask.java | 4 ++ .../AlarmsUnassignHousekeeperTask.java | 4 ++ .../EntitiesDeletionHousekeeperTask.java | 4 ++ .../data/housekeeper/HousekeeperTask.java | 4 ++ .../LatestTsDeletionHousekeeperTask.java | 5 ++ ...TenantEntitiesDeletionHousekeeperTask.java | 5 ++ .../TsHistoryDeletionHousekeeperTask.java | 5 ++ .../server/dao/ota/BaseOtaPackageService.java | 48 ++++++++++++------- .../server/dao/ota/OtaPackageDao.java | 7 ++- .../server/dao/sql/ota/JpaOtaPackageDao.java | 10 ++++ .../dao/sql/ota/OtaPackageRepository.java | 9 +++- .../dao/service/OtaPackageServiceTest.java | 37 ++++++++++++++ 13 files changed, 128 insertions(+), 24 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index cf37afad5c..0c12767b60 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -23,8 +23,8 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; @@ -127,10 +127,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. }) public class HousekeeperServiceTest extends AbstractControllerTest { - @SpyBean + @MockitoSpyBean private HousekeeperService housekeeperService; - @SpyBean + @MockitoSpyBean private HousekeeperReprocessingService housekeeperReprocessingService; + @MockitoSpyBean + private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; @Autowired private EventService eventService; @Autowired @@ -153,8 +155,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private CustomerService customerService; @Autowired private DashboardService dashboardService; - @SpyBean - private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; private TenantId tenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java index dea590295e..d66ec046de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9214680001573764374L; + private List alarms; public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 0313190056..445850d387 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -33,6 +34,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9156667024462937756L; + private String userTitle; private List alarms; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index fe25a98a1d..c1a233b1c4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9009068831061529286L; + private EntityType entityType; private List entities; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 875ef2765f..2df7cf4dd4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.io.Serializable; @JsonIgnoreProperties(ignoreUnknown = true) @@ -45,6 +46,9 @@ import java.io.Serializable; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class HousekeeperTask implements Serializable { + @Serial + private static final long serialVersionUID = -2585974110832225152L; + private TenantId tenantId; private EntityId entityId; private HousekeeperTaskType taskType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java index cd3e94e5c6..931c2931cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 5193191938513490138L; + private String key; public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java index 443d929d8b..be7ff6f7ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = -8033108795318393447L; + private EntityType entityType; public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java index d9315f0ff4..b520899ca4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 4573851542705079043L; + private String key; public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 95515bd15e..aac4157706 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -196,7 +196,9 @@ public class BaseOtaPackageService extends AbstractCachedEntityService INCORRECT_OTA_PACKAGE_ID + id); try { + Long oid = otaPackageDao.getDataOidById(otaPackageId.getId()); otaPackageDao.removeById(tenantId, otaPackageId.getId()); + unlinkDataIfPresent(tenantId, otaPackageId, oid); publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(otaPackageId).build()); } catch (Exception t) { @@ -215,6 +217,20 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = - new PaginatedRemover<>() { - - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); - } + private final PaginatedRemover tenantOtaPackageRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { - deleteOtaPackage(tenantId, entity.getId()); - } - }; + @Override + protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { + deleteOtaPackage(tenantId, entity.getId()); + } + }; @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index c11a13cbe1..875aaea4ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -18,15 +18,20 @@ package org.thingsboard.server.dao.ota; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; +import java.util.UUID; + public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, ExportableEntityDao { Long sumDataSizeByTenantId(TenantId tenantId); OtaPackage findOtaPackageByTenantIdAndTitleAndVersion(TenantId tenantId, String title, String version); + Long getDataOidById(UUID id); + + Integer unlinkLargeObject(Long dataOid); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index d67110d53d..76bf5ca26e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -77,6 +77,16 @@ public class JpaOtaPackageDao extends JpaAbstractDao findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + @Query(value = "SELECT data FROM ota_package WHERE id = :id AND data IS NOT NULL", nativeQuery = true) + Long getDataOidById(@Param("id") UUID id); + + @Transactional + @Query(value = "SELECT lo_unlink(:oid)", nativeQuery = true) + Integer unlinkLargeObject(@Param("oid") Long oid); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java index a38499c82c..e79a7bad07 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -77,6 +78,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + OtaPackageDao otaPackageDao; + @Autowired TbTenantProfileCache tenantProfileCache; @Before @@ -533,6 +536,39 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertNull(foundFirmware); } + @Test + public void testDeleteOtaPackageWithoutData() { + OtaPackageInfo firmwareInfo = new OtaPackageInfo(); + firmwareInfo.setTenantId(tenantId); + firmwareInfo.setDeviceProfileId(deviceProfileId); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle(TITLE); + firmwareInfo.setVersion(VERSION); + OtaPackageInfo savedFirmwareInfo = otaPackageService.saveOtaPackageInfo(firmwareInfo, false); + + Assert.assertNotNull(savedFirmwareInfo); + Assert.assertNotNull(savedFirmwareInfo.getId()); + + // Should not throw NPE when deleting package without data (OID is null) + otaPackageService.deleteOtaPackage(tenantId, savedFirmwareInfo.getId()); + + OtaPackageInfo foundFirmware = otaPackageService.findOtaPackageInfoById(tenantId, savedFirmwareInfo.getId()); + Assert.assertNull(foundFirmware); + } + + @Test + public void testDeleteOtaPackageUnlinksLargeObject() { + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); + + Long oid = otaPackageDao.getDataOidById(savedFirmware.getId().getId()); + Assert.assertNotNull(oid); + + otaPackageService.deleteOtaPackage(tenantId, savedFirmware.getId()); + + // Verify the large object was unlinked - PostgreSQL throws an exception when the object doesn't exist + assertThatThrownBy(() -> otaPackageDao.unlinkLargeObject(oid)).hasMessageContaining("large object " + oid + " does not exist"); + } + @Test public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); @@ -726,4 +762,5 @@ public class OtaPackageServiceTest extends AbstractServiceTest { firmware.setDataSize(DATA_SIZE); return firmware; } + } From cc6af84a928711c5d1d9f20e03e1661419969b7d Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 28 Jan 2026 12:04:42 +0200 Subject: [PATCH 03/71] Small revert --- .../server/dao/ota/BaseOtaPackageService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index aac4157706..54577ff9eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -196,7 +196,7 @@ public class BaseOtaPackageService extends AbstractCachedEntityService INCORRECT_OTA_PACKAGE_ID + id); try { - Long oid = otaPackageDao.getDataOidById(otaPackageId.getId()); + Long oid = getDataOidById(tenantId, otaPackageId); otaPackageDao.removeById(tenantId, otaPackageId.getId()); unlinkDataIfPresent(tenantId, otaPackageId, oid); publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); @@ -217,6 +217,16 @@ public class BaseOtaPackageService extends AbstractCachedEntityService Date: Wed, 4 Mar 2026 14:44:54 +0100 Subject: [PATCH 04/71] Fix LwM2M Redis stores using separate connections for SCAN and GET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using the same connection for both SCAN cursor iteration and GET value fetches causes Jedis 5.x response-ordering corruption: the SCAN response parser receives a GET response (byte[]) where it expects a List, and vice versa, resulting in ClassCastException on startup. Fix: open two connections per getAll() call — one dedicated to the scan cursor and one for value fetches — eliminating any interleaving. Affected: TbRedisLwM2MClientStore, TbRedisLwM2MModelConfigStore, TbLwM2mRedisRegistrationStore. Co-Authored-By: Claude Sonnet 4.6 --- .../store/TbLwM2mRedisRegistrationStore.java | 18 ++- .../server/store/TbRedisLwM2MClientStore.java | 14 +- .../store/TbRedisLwM2MModelConfigStore.java | 18 ++- .../store/TbRedisLwM2MClientStoreTest.java | 137 ++++++++++++++++++ 4 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java index 4b2ef07994..8158e82a81 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java @@ -323,22 +323,24 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab @Override public Iterator getAllRegistrations() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Collection list = new LinkedList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(REG_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - list.add(deserializeReg(element)); + byte[] element = getConnection.get(key); + if (element != null) { + list.add(deserializeReg(element)); + } }); }); return list.iterator(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java index 3293cd8b53..4beefb4896 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java @@ -61,21 +61,21 @@ public class TbRedisLwM2MClientStore implements TbLwM2MClientStore { @Override public Set getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Set clients = new HashSet<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(CLIENT_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); + byte[] element = getConnection.get(key); if (element != null) { try { clients.add(deserialize(element)); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java index 73b6f3c8df..31a78234d0 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java @@ -35,22 +35,24 @@ public class TbRedisLwM2MModelConfigStore implements TbLwM2MModelConfigStore { @Override public List getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { List configs = new ArrayList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(MODEL_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + byte[] element = getConnection.get(key); + if (element != null) { + configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + } }); }); return configs; diff --git a/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java new file mode 100644 index 0000000000..9fe29c1188 --- /dev/null +++ b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.store; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2MClientState; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; + +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.transport.lwm2m.server.store.util.LwM2MClientSerDes.serialize; + +/** + * Verifies that {@link TbRedisLwM2MClientStore#getAll()} uses separate connections for + * SCAN and GET operations to prevent Jedis 5.x response-ordering corruption that occurs + * when both commands share the same connection. + */ +@ExtendWith(MockitoExtension.class) +class TbRedisLwM2MClientStoreTest { + + @Mock + RedisConnectionFactory connectionFactory; + + @Mock + RedisConnection scanConnection; + + @Mock + RedisConnection getConnection; + + TbRedisLwM2MClientStore store; + + @BeforeEach + void setUp() { + // First getConnection() call → scanConnection, second → getConnection + when(connectionFactory.getConnection()) + .thenReturn(scanConnection) + .thenReturn(getConnection); + store = new TbRedisLwM2MClientStore(connectionFactory); + } + + @Test + void getAll_returnsSingleClient() { + LwM2mClient client = new LwM2mClient("nodeId", "testEndpoint"); + client.setState(LwM2MClientState.REGISTERED); + byte[] key = "CLIENT#EP#testEndpoint".getBytes(); + byte[] value = serialize(client); + + // Cursor created before thenReturn to avoid Mockito unfinished-stubbing error + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + when(getConnection.get(key)).thenReturn(value); + + Set result = store.getAll(); + + assertThat(result).hasSize(1); + assertThat(result.iterator().next().getEndpoint()).isEqualTo("testEndpoint"); + } + + @Test + void getAll_getIsNeverCalledOnScanConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(scanConnection, never()).get(any(byte[].class)); + } + + @Test + void getAll_scanIsNeverCalledOnGetConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(getConnection, never()).scan(any(ScanOptions.class)); + } + + @Test + void getAll_skipsKeyWhenValueIsNull() { + byte[] key = "CLIENT#EP#gone".getBytes(); + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + // getConnection.get(key) returns null by default — no stubbing needed + + Set result = store.getAll(); + + assertThat(result).isEmpty(); + } + + /** + * Creates a mock {@link Cursor} that iterates over the given keys via {@code forEachRemaining}. + * The cursor is created separately (not inside a {@code thenReturn()} argument) to avoid + * Mockito's "unfinished stubbing" error caused by nested {@code when()} calls. + */ + @SuppressWarnings("unchecked") + private static Cursor cursorOf(byte[]... keys) { + Cursor cursor = mock(Cursor.class); + List keyList = List.of(keys); + doAnswer(inv -> { + Consumer action = inv.getArgument(0); + keyList.forEach(action); + return null; + }).when(cursor).forEachRemaining(any(Consumer.class)); + return cursor; + } +} From 455f62eaefcbf7f996af54f1d7da33238f797601 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 16:19:59 +0100 Subject: [PATCH 05/71] fix: delay LwM2M client stop in FW update to prevent Execute response race When Execute RPC is sent to FW Update resource (/5/0/2), the test client's startUpdating() scheduled the client stop with 0 delay. This caused a race where the client stopped before the CoAP Execute response (2.04 Changed) was delivered to the server, resulting in RequestCanceledException and INTERNAL_SERVER_ERROR instead of CHANGED. Adding a 1-second delay before leshanClient.stop() ensures the CoAP response is transmitted and received before the client disconnects, fixing the flaky testExecuteUpdateFWById_Result_CHANGED test. Co-Authored-By: Claude Sonnet 4.6 --- .../server/transport/lwm2m/client/FwLwM2MDevice.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java index 9bed9bc483..6a7d631eb1 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java @@ -193,7 +193,7 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { } catch (Exception e) { log.error("Error during firmware update", e); } - }, 0, TimeUnit.SECONDS); // start immediately, without further delay + }, 1, TimeUnit.SECONDS); // delay 1 sec to allow CoAP Execute response to be delivered before client stops } protected void setLeshanClient(LeshanClient leshanClient) { From 30b046fa30ce6d6c3b490733a78d3bb6f769d779 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Mar 2026 17:53:27 +0200 Subject: [PATCH 06/71] sending ws error when the telemetry queries exceed limit --- ...efaultTbEntityDataSubscriptionService.java | 19 +++++++- .../server/controller/WebsocketApiTest.java | 46 +++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index f0f2e1eaa7..71df830f8d 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -34,6 +34,7 @@ import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -242,7 +243,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc @Override public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId()); + log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + finalCtx.sendWsMsg(new EntityDataUpdate(finalCtx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + } } }, wsCallBackExecutor); } @@ -258,7 +262,18 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc handleLatestCmd(ctx, cmd.getLatestCmd()); } if (cmd.getTsCmd() != null) { - handleTimeSeriesCmd(ctx, cmd.getTsCmd()); + Futures.addCallback(handleTimeSeriesCmd(ctx, cmd.getTsCmd()), new FutureCallback<>() { + @Override + public void onSuccess(TbEntityDataSubCtx result) {} + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] Failed to process timeseries command", ctx.getSessionId(), ctx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + } + } + }, wsCallBackExecutor); } } else { checkAndSendInitialData(ctx); diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 87ba0ec3e8..aa7ffaeaf5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; @@ -60,7 +63,9 @@ import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.service.subscription.SubscriptionErrorCode; import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -95,6 +100,9 @@ public class WebsocketApiTest extends AbstractControllerTest { @Autowired private TelemetrySubscriptionService tsService; + @SpyBean + private TimeseriesService timeseriesService; + Device device; DeviceTypeFilter dtf; @@ -965,6 +973,44 @@ public class WebsocketApiTest extends AbstractControllerTest { } + @Test + public void testHistoryCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + // Register for 2 messages: initial entity page data + error + getWsClient().registerWaitForUpdate(2); + getWsClient().sendHistoryCmd(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + getWsClient().waitForUpdate(); + + EntityDataUpdate errorUpdate = JacksonUtil.fromString(getWsClient().getLastMsg(), EntityDataUpdate.class); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + + @Test + public void testTimeSeriesCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + // Register for 2 messages: initial entity page data + error + getWsClient().registerWaitForUpdate(2); + getWsClient().subscribeTsUpdate(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + getWsClient().waitForUpdate(); + + EntityDataUpdate errorUpdate = JacksonUtil.fromString(getWsClient().getLastMsg(), EntityDataUpdate.class); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + private void sendTelemetry(Device device, List tsData) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); tsService.saveTimeseries(TimeseriesSaveRequest.builder() From bc39695fabe5441ce1a18861664bcd3d86aead74 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Mar 2026 18:24:39 +0200 Subject: [PATCH 07/71] test fixes --- .../server/controller/WebsocketApiTest.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index aa7ffaeaf5..eab878854d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -982,12 +982,7 @@ public class WebsocketApiTest extends AbstractControllerTest { List keys = List.of("temperature"); long now = System.currentTimeMillis(); - // Register for 2 messages: initial entity page data + error - getWsClient().registerWaitForUpdate(2); - getWsClient().sendHistoryCmd(keys, now, TimeUnit.HOURS.toMillis(1), dtf); - getWsClient().waitForUpdate(); - - EntityDataUpdate errorUpdate = JacksonUtil.fromString(getWsClient().getLastMsg(), EntityDataUpdate.class); + EntityDataUpdate errorUpdate = getWsClient().sendHistoryCmd(keys, now, TimeUnit.HOURS.toMillis(1), dtf); assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); } @@ -1001,12 +996,7 @@ public class WebsocketApiTest extends AbstractControllerTest { List keys = List.of("temperature"); long now = System.currentTimeMillis(); - // Register for 2 messages: initial entity page data + error - getWsClient().registerWaitForUpdate(2); - getWsClient().subscribeTsUpdate(keys, now, TimeUnit.HOURS.toMillis(1), dtf); - getWsClient().waitForUpdate(); - - EntityDataUpdate errorUpdate = JacksonUtil.fromString(getWsClient().getLastMsg(), EntityDataUpdate.class); + EntityDataUpdate errorUpdate = getWsClient().subscribeTsUpdate(keys, now, TimeUnit.HOURS.toMillis(1), dtf); assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); } From 63401c5f1b964ec739046e48c48bdc6c295c687d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 19:30:28 +0100 Subject: [PATCH 08/71] Fix flaky AuditLogControllerTest#testAuditLogsSysAdmin by using Awaitility Audit logs are saved asynchronously via executor.submit() in AuditLogServiceImpl, so querying immediately after tenant profile creation could miss logs not yet persisted. Replace direct assertEquals with Awaitility.await().atMost(10s).untilAsserted() to tolerate the async save delay. Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/AuditLogControllerTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index af5df3d733..728dc1ec0e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -136,9 +137,10 @@ public class AuditLogControllerTest extends AbstractControllerTest { doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); } - List loadedAuditLogs = getAuditLogs(100, "/api/audit/logs?"); - - Assert.assertEquals("Have X audit log before this test + New tenant profiles in the test", loadedAuditLogsBefore.size() + 3, loadedAuditLogs.size()); + int expectedSize = loadedAuditLogsBefore.size() + 3; + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> + Assert.assertEquals("Have X audit log before this test + New tenant profiles in the test", + expectedSize, getAuditLogs(100, "/api/audit/logs?").size())); } private List getAuditLogs(int pageSize, String urlTemplate) throws Exception { From 2cd0a746a3e45496338af4d686f89ead181641b8 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 6 Mar 2026 11:15:59 +0200 Subject: [PATCH 09/71] Refactoring --- .../server/dao/ota/BaseOtaPackageService.java | 4 ++-- .../server/dao/sql/ota/OtaPackageRepository.java | 3 ++- .../server/dao/service/OtaPackageServiceTest.java | 13 +++++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 54577ff9eb..c43274cc26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -234,10 +234,10 @@ public class BaseOtaPackageService extends AbstractCachedEntityService findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + // The 'data' column is of type OID (PostgreSQL large object reference), so it returns the OID as Long @Query(value = "SELECT data FROM ota_package WHERE id = :id AND data IS NOT NULL", nativeQuery = true) Long getDataOidById(@Param("id") UUID id); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java index e79a7bad07..2f296d38f2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java @@ -44,7 +44,6 @@ import org.thingsboard.server.dao.tenant.TenantProfileService; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -121,10 +120,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; - List packages = new ArrayList<>(maxSumDataSize); - for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createAndSaveFirmware(tenantId, "0." + i)); + createAndSaveFirmware(tenantId, "0." + i); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -603,8 +600,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); @@ -658,8 +655,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); From 04aed068bf39fbed99d1cf8f5ae6d6961b33464b Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 6 Mar 2026 11:40:54 +0200 Subject: [PATCH 10/71] Edge api key sync (#87) Add bidirectional edge sync for ApiKey entity --- .../service/edge/EdgeContextComponent.java | 8 + .../service/edge/EdgeMsgConstructorUtils.java | 16 ++ .../service/edge/rpc/EdgeGrpcSession.java | 11 + .../edge/rpc/processor/BaseEdgeProcessor.java | 2 +- .../processor/apikey/ApiKeyEdgeProcessor.java | 105 ++++++++ .../rpc/processor/apikey/ApiKeyProcessor.java | 28 +++ .../processor/apikey/BaseApiKeyProcessor.java | 63 +++++ .../rpc/processor/user/UserEdgeProcessor.java | 7 + .../server/edge/ApiKeyEdgeTest.java | 238 ++++++++++++++++++ .../server/dao/pat/ApiKeyService.java | 8 + .../common/data/edge/EdgeEventType.java | 3 +- .../common/data/id/EntityIdFactory.java | 1 + common/edge-api/src/main/proto/edge.proto | 9 + .../thingsboard/server/dao/pat/ApiKeyDao.java | 7 + .../server/dao/pat/ApiKeyServiceImpl.java | 52 ++++ .../server/dao/sql/pat/ApiKeyRepository.java | 7 + .../server/dao/sql/pat/JpaApiKeyDao.java | 13 + 17 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java create mode 100644 application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index c03c6affe2..c99261b249 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -25,6 +25,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.dao.ai.AiModelService; +import org.thingsboard.server.dao.pat.ApiKeyService; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetProfileService; @@ -61,6 +62,7 @@ import org.thingsboard.server.service.edge.rpc.EdgeEventStorageSettings; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.ai.AiModelProcessor; +import org.thingsboard.server.service.edge.rpc.processor.apikey.ApiKeyProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.comment.AlarmCommentProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessor; @@ -273,6 +275,12 @@ public class EdgeContextComponent { @Autowired private AiModelProcessor aiModelProcessor; + @Autowired + private ApiKeyService apiKeyService; + + @Autowired + private ApiKeyProcessor apiKeyProcessor; + @Autowired private UserProcessor userProcessor; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java index 445741fd50..01d2178cd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java @@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.ai.AiModel; +import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.asset.Asset; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AiModelId; +import org.thingsboard.server.common.data.id.ApiKeyId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -98,6 +100,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -741,6 +744,19 @@ public class EdgeMsgConstructorUtils { .setIdLSB(aiModelId.getId().getLeastSignificantBits()).build(); } + public static ApiKeyUpdateMsg constructApiKeyUpdatedMsg(UpdateMsgType msgType, ApiKey apiKey) { + return ApiKeyUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(apiKey)) + .setIdMSB(apiKey.getId().getId().getMostSignificantBits()) + .setIdLSB(apiKey.getId().getId().getLeastSignificantBits()).build(); + } + + public static ApiKeyUpdateMsg constructApiKeyDeleteMsg(ApiKeyId apiKeyId) { + return ApiKeyUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(apiKeyId.getId().getMostSignificantBits()) + .setIdLSB(apiKeyId.getId().getLeastSignificantBits()).build(); + } + public static List mergeAndFilterDownlinkDuplicates(List edgeEvents) { try { edgeEvents = removeDownlinkDuplicates(edgeEvents); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 474a6860d4..de4a92ae8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.dao.edge.stats.EdgeStatsKey; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -988,6 +989,16 @@ public abstract class EdgeGrpcSession implements Closeable { } } } + if (uplinkMsg.getApiKeyUpdateMsgCount() > 0) { + for (ApiKeyUpdateMsg apiKeyUpdateMsg : uplinkMsg.getApiKeyUpdateMsgList()) { + sequenceDependencyLock.lock(); + try { + result.add(ctx.getApiKeyProcessor().processApiKeyMsgFromEdge(edge.getTenantId(), edge, apiKeyUpdateMsg)); + } finally { + sequenceDependencyLock.unlock(); + } + } + } } catch (Exception e) { String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); log.trace("[{}][{}] Can't process uplink msg [{}]", tenantId, edge.getId(), uplinkMsg, e); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 207bbc2ce8..8455d86833 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -146,7 +146,7 @@ public abstract class BaseEdgeProcessor implements EdgeProcessor { UPDATED_COMMENT, DELETED -> true; default -> switch (type) { case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, - WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, CALCULATED_FIELD, AI_MODEL, NOTIFICATION_TEMPLATE, + WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, CALCULATED_FIELD, AI_MODEL, API_KEY, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, NOTIFICATION_RULE -> true; default -> false; }; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java new file mode 100644 index 0000000000..960c0dd695 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.exception.DataValidationException; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.EdgeVersion; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; + +import java.util.UUID; + +@Slf4j +@Component +@TbCoreComponent +public class ApiKeyEdgeProcessor extends BaseApiKeyProcessor implements ApiKeyProcessor { + + @Override + public ListenableFuture processApiKeyMsgFromEdge(TenantId tenantId, Edge edge, ApiKeyUpdateMsg apiKeyUpdateMsg) { + ApiKeyId apiKeyId = new ApiKeyId(new UUID(apiKeyUpdateMsg.getIdMSB(), apiKeyUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getEdgeId().set(edge.getId()); + + return switch (apiKeyUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE, ENTITY_UPDATED_RPC_MESSAGE -> { + boolean created = saveOrUpdateApiKey(tenantId, apiKeyId, apiKeyUpdateMsg); + if (created) { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (apiKey != null) { + pushEntityEventToRuleEngine(tenantId, edge, apiKey, TbMsgType.ENTITY_CREATED); + } + } + yield Futures.immediateFuture(null); + } + case ENTITY_DELETED_RPC_MESSAGE -> { + deleteApiKey(tenantId, edge, apiKeyId); + yield Futures.immediateFuture(null); + } + default -> handleUnsupportedMsgType(apiKeyUpdateMsg.getMsgType()); + }; + } catch (DataValidationException e) { + return Futures.immediateFailedFuture(e); + } finally { + edgeSynchronizationManager.getEdgeId().remove(); + } + } + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + ApiKeyId apiKeyId = new ApiKeyId(edgeEvent.getEntityId()); + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(edgeEvent.getTenantId(), apiKeyId); + if (apiKey != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + ApiKeyUpdateMsg apiKeyUpdateMsg = EdgeMsgConstructorUtils.constructApiKeyUpdatedMsg(msgType, apiKey); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addApiKeyUpdateMsg(apiKeyUpdateMsg) + .build(); + } + } + case DELETED -> { + ApiKeyUpdateMsg apiKeyUpdateMsg = EdgeMsgConstructorUtils.constructApiKeyDeleteMsg(apiKeyId); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addApiKeyUpdateMsg(apiKeyUpdateMsg) + .build(); + } + } + return null; + } + + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.API_KEY; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java new file mode 100644 index 0000000000..f50594c9b7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; + +public interface ApiKeyProcessor extends EdgeProcessor { + + ListenableFuture processApiKeyMsgFromEdge(TenantId tenantId, Edge edge, ApiKeyUpdateMsg apiKeyUpdateMsg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java new file mode 100644 index 0000000000..c90ee572bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +public abstract class BaseApiKeyProcessor extends BaseEdgeProcessor { + + protected boolean saveOrUpdateApiKey(TenantId tenantId, ApiKeyId apiKeyId, ApiKeyUpdateMsg apiKeyUpdateMsg) { + boolean isCreated = false; + try { + ApiKey apiKey = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + if (apiKey == null) { + throw new RuntimeException("[{" + tenantId + "}] apiKeyUpdateMsg {" + apiKeyUpdateMsg + " } cannot be converted to apiKey"); + } + + ApiKey existingApiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (existingApiKey == null) { + apiKey.setCreatedTime(Uuids.unixTimestamp(apiKeyId.getId())); + isCreated = true; + } + + apiKey.setId(apiKeyId); + edgeCtx.getApiKeyService().saveApiKey(tenantId, apiKey); + } catch (Exception e) { + log.error("[{}] Failed to process apiKey update msg [{}]", tenantId, apiKeyUpdateMsg, e); + throw e; + } + return isCreated; + } + + protected void deleteApiKey(TenantId tenantId, Edge edge, ApiKeyId apiKeyId) { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (apiKey != null) { + edgeCtx.getApiKeyService().deleteApiKey(tenantId, apiKey, false); + pushEntityEventToRuleEngine(tenantId, edge, apiKey, TbMsgType.ENTITY_DELETED); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java index 027b1225c4..03fc29ae3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; @@ -34,6 +35,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.exception.DataValidationException; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; @@ -42,6 +44,7 @@ import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import java.util.List; import java.util.UUID; @Slf4j @@ -135,6 +138,10 @@ public class UserEdgeProcessor extends BaseUserProcessor implements UserProcesso if (userCredentialsByUserId != null) { builder.addUserCredentialsUpdateMsg(EdgeMsgConstructorUtils.constructUserCredentialsUpdatedMsg(userCredentialsByUserId)); } + List apiKeys = edgeCtx.getApiKeyService().findApiKeysByUserId(edgeEvent.getTenantId(), userId); + for (ApiKey apiKey : apiKeys) { + builder.addApiKeyUpdateMsg(EdgeMsgConstructorUtils.constructApiKeyUpdatedMsg(msgType, apiKey)); + } return builder.build(); } } diff --git a/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java new file mode 100644 index 0000000000..bd46cb5251 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edge; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.common.data.pat.ApiKeyInfo; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.pat.ApiKeyService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.gen.edge.v1.UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; + +@DaoSqlTest +public class ApiKeyEdgeTest extends AbstractEdgeTest { + + @Autowired + private ApiKeyService apiKeyService; + + private static final String DEFAULT_API_KEY_DESCRIPTION = "Edge Test ApiKey"; + private static final String UPDATED_API_KEY_DESCRIPTION = "Updated Edge Test ApiKey"; + + @Test + public void testApiKey_create_update_delete_fromCloud() throws Exception { + // create ApiKey + ApiKeyInfo apiKeyInfo = createSimpleApiKeyInfo(DEFAULT_API_KEY_DESCRIPTION); + + edgeImitator.expectMessageAmount(1); + ApiKey savedApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + ApiKeyUpdateMsg apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(savedApiKey.getUuidId().getMostSignificantBits(), apiKeyUpdateMsg.getIdMSB()); + Assert.assertEquals(savedApiKey.getUuidId().getLeastSignificantBits(), apiKeyUpdateMsg.getIdLSB()); + ApiKey apiKeyFromMsg = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + Assert.assertNotNull(apiKeyFromMsg); + + Assert.assertEquals(DEFAULT_API_KEY_DESCRIPTION, apiKeyFromMsg.getDescription()); + Assert.assertEquals(savedApiKey.getTenantId(), apiKeyFromMsg.getTenantId()); + + // update ApiKey + edgeImitator.expectMessageAmount(1); + savedApiKey.setDescription(UPDATED_API_KEY_DESCRIPTION); + savedApiKey = doPost("/api/apiKey", new ApiKeyInfo(savedApiKey), ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + apiKeyFromMsg = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + Assert.assertNotNull(apiKeyFromMsg); + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(UPDATED_API_KEY_DESCRIPTION, apiKeyFromMsg.getDescription()); + + // delete ApiKey + edgeImitator.expectMessageAmount(1); + doDelete("/api/apiKey/" + savedApiKey.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(savedApiKey.getUuidId().getMostSignificantBits(), apiKeyUpdateMsg.getIdMSB()); + Assert.assertEquals(savedApiKey.getUuidId().getLeastSignificantBits(), apiKeyUpdateMsg.getIdLSB()); + } + + @Test + public void testApiKey_create_update_delete_toCloud() throws Exception { + // create + ApiKey apiKey = createSimpleApiKey(DEFAULT_API_KEY_DESCRIPTION); + UUID uuid = Uuids.timeBased(); + UplinkMsg uplinkMsg = getUplinkMsg(uuid, apiKey, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + + checkApiKeyOnCloud(uplinkMsg, uuid, apiKey.getDescription()); + + // update + apiKey.setDescription(UPDATED_API_KEY_DESCRIPTION); + UplinkMsg updatedUplinkMsg = getUplinkMsg(uuid, apiKey, UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + + checkApiKeyOnCloud(updatedUplinkMsg, uuid, apiKey.getDescription()); + + // delete + UplinkMsg deleteUplinkMsg = getDeleteUplinkMsg(uuid); + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(deleteUplinkMsg); + Assert.assertTrue(edgeImitator.waitForResponses()); + + ApiKeyId apiKeyId = new ApiKeyId(uuid); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + Assert.assertNull(apiKeyService.findApiKeyById(tenantId, apiKeyId)) + ); + } + + @Test + public void testApiKey_pushedDuringUserSync() throws Exception { + // create tenant admin user - expect 3 messages: 1 UserUpdateMsg + 2 UserCredentialsUpdateMsg + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setTenantId(tenantId); + user.setEmail("apiKeyTestUser@thingsboard.org"); + user.setFirstName("ApiKey"); + user.setLastName("TestUser"); + + edgeImitator.expectMessageAmount(3); + User savedUser = createUser(user, "tenant"); + Assert.assertTrue(edgeImitator.waitForMessages()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserUpdateMsg.class).size()); + Assert.assertEquals(2, edgeImitator.findAllMessagesByType(UserCredentialsUpdateMsg.class).size()); + + // create API key for this user - expect 1 ApiKeyUpdateMsg + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setTenantId(tenantId); + apiKeyInfo.setUserId(savedUser.getId()); + apiKeyInfo.setDescription("Test API Key for user sync"); + apiKeyInfo.setEnabled(true); + + edgeImitator.expectMessageAmount(1); + doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(ApiKeyUpdateMsg.class).size()); + + // update user - expect 3 messages: UserUpdateMsg + UserCredentialsUpdateMsg + ApiKeyUpdateMsg + savedUser.setLastName("UpdatedLastName"); + edgeImitator.expectMessageAmount(3); + doPost("/api/user", savedUser, User.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserUpdateMsg.class).size()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserCredentialsUpdateMsg.class).size()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(ApiKeyUpdateMsg.class).size()); + + Optional apiKeyUpdateMsgOpt = edgeImitator.findMessageByType(ApiKeyUpdateMsg.class); + Assert.assertTrue(apiKeyUpdateMsgOpt.isPresent()); + ApiKeyUpdateMsg apiKeyUpdateMsg = apiKeyUpdateMsgOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + } + + private ApiKeyInfo createSimpleApiKeyInfo(String description) { + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setTenantId(tenantId); + apiKeyInfo.setUserId(tenantAdminUserId); + apiKeyInfo.setDescription(description); + apiKeyInfo.setEnabled(true); + return apiKeyInfo; + } + + private ApiKey createSimpleApiKey(String description) { + ApiKey apiKey = new ApiKey(); + apiKey.setTenantId(tenantId); + apiKey.setUserId(tenantAdminUserId); + apiKey.setDescription(description); + apiKey.setEnabled(true); + apiKey.setValue("test-api-key-value-" + UUID.randomUUID()); + return apiKey; + } + + private UplinkMsg getDeleteUplinkMsg(UUID uuid) throws InvalidProtocolBufferException { + UplinkMsg.Builder upLinkMsgBuilder = UplinkMsg.newBuilder(); + ApiKeyUpdateMsg.Builder apiKeyDeleteMsgBuilder = ApiKeyUpdateMsg.newBuilder(); + apiKeyDeleteMsgBuilder.setMsgType(ENTITY_DELETED_RPC_MESSAGE); + apiKeyDeleteMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + apiKeyDeleteMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + testAutoGeneratedCodeByProtobuf(apiKeyDeleteMsgBuilder); + + upLinkMsgBuilder.addApiKeyUpdateMsg(apiKeyDeleteMsgBuilder.build()); + testAutoGeneratedCodeByProtobuf(upLinkMsgBuilder); + + return upLinkMsgBuilder.build(); + } + + private UplinkMsg getUplinkMsg(UUID uuid, ApiKey apiKey, UpdateMsgType updateMsgType) throws InvalidProtocolBufferException { + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + ApiKeyUpdateMsg.Builder apiKeyUpdateMsgBuilder = ApiKeyUpdateMsg.newBuilder(); + apiKeyUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + apiKeyUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + apiKeyUpdateMsgBuilder.setEntity(JacksonUtil.toString(apiKey)); + apiKeyUpdateMsgBuilder.setMsgType(updateMsgType); + testAutoGeneratedCodeByProtobuf(apiKeyUpdateMsgBuilder); + uplinkMsgBuilder.addApiKeyUpdateMsg(apiKeyUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + return uplinkMsgBuilder.build(); + } + + private void checkApiKeyOnCloud(UplinkMsg uplinkMsg, UUID uuid, String description) throws Exception { + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsg); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + ApiKey apiKey = apiKeyService.findApiKeyById(tenantId, new ApiKeyId(uuid)); + Assert.assertNotNull(apiKey); + Assert.assertEquals(description, apiKey.getDescription()); + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java index 798549cf85..19c17127e3 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java @@ -24,10 +24,14 @@ import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKeyInfo; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.List; + public interface ApiKeyService extends EntityDaoService { ApiKey saveApiKey(TenantId tenantId, ApiKeyInfo apiKey); + ApiKey saveApiKey(TenantId tenantId, ApiKey apiKey); + void deleteApiKey(TenantId tenantId, ApiKey apiKey, boolean force); void deleteByUserId(TenantId tenantId, UserId userId); @@ -38,4 +42,8 @@ public interface ApiKeyService extends EntityDaoService { PageData findApiKeysByUserId(TenantId tenantId, UserId userId, PageLink pageLink); + List findApiKeysByUserId(TenantId tenantId, UserId userId); + + PageData findApiKeysByTenantId(TenantId tenantId, PageLink pageLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java index 834e19a1c2..474a0270c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java @@ -48,7 +48,8 @@ public enum EdgeEventType { OAUTH2_CLIENT(true, EntityType.OAUTH2_CLIENT), DOMAIN(true, EntityType.DOMAIN), CALCULATED_FIELD(false, EntityType.CALCULATED_FIELD), - AI_MODEL(true, EntityType.AI_MODEL); + AI_MODEL(true, EntityType.AI_MODEL), + API_KEY(true, EntityType.API_KEY); private final boolean allEdgesRelated; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 7baa8e72f2..ae033f769c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -114,6 +114,7 @@ public class EntityIdFactory { case DOMAIN -> new DomainId(uuid); case CALCULATED_FIELD -> new CalculatedFieldId(uuid); case AI_MODEL -> new AiModelId(uuid); + case API_KEY -> new ApiKeyId(uuid); case ADMIN_SETTINGS -> new AdminSettingsId(uuid); default -> throw new IllegalArgumentException("EdgeEventType " + edgeEventType + " is not supported!"); }; diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 913a9199e7..ea45efaf7c 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -145,6 +145,13 @@ message AiModelUpdateMsg{ string entity = 4; } +message ApiKeyUpdateMsg{ + UpdateMsgType msgType = 1; + int64 idMSB = 2; + int64 idLSB = 3; + string entity = 4; +} + message EntityDataProto { int64 entityIdMSB = 1; int64 entityIdLSB = 2; @@ -455,6 +462,7 @@ message UplinkMsg { repeated AiModelUpdateMsg aiModelUpdateMsg = 27; repeated UserUpdateMsg userUpdateMsg = 28; repeated UserCredentialsUpdateMsg userCredentialsUpdateMsg = 29; + repeated ApiKeyUpdateMsg apiKeyUpdateMsg = 30; } message UplinkResponseMsg { @@ -506,4 +514,5 @@ message DownlinkMsg { repeated OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = 34; repeated CalculatedFieldUpdateMsg calculatedFieldUpdateMsg = 35; repeated AiModelUpdateMsg aiModelUpdateMsg = 36; + repeated ApiKeyUpdateMsg apiKeyUpdateMsg = 37; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java index d71d6502cf..f5ff79fb52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java @@ -17,15 +17,22 @@ package org.thingsboard.server.dao.pat; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.dao.Dao; +import java.util.List; import java.util.Set; public interface ApiKeyDao extends Dao { ApiKey findByValue(String value); + PageData findByTenantId(TenantId tenantId, PageLink pageLink); + + List findByTenantIdAndUserId(TenantId tenantId, UserId userId); + Set deleteByTenantId(TenantId tenantId); Set deleteByUserId(TenantId tenantId, UserId userId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java index 17a3e2bda3..c50e13b633 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.exception.DataValidationException; import org.thingsboard.server.common.data.id.ApiKeyId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -34,9 +35,11 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKeyInfo; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.validator.ApiKeyDataValidator; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -95,6 +98,27 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService findApiKeysByUserId(TenantId tenantId, UserId userId) { + log.trace("Executing findApiKeysByUserId [{}][{}]", tenantId, userId); + validateId(userId, id -> INCORRECT_USER_ID + id); + return apiKeyDao.findByTenantIdAndUserId(tenantId, userId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findApiKeyById(tenantId, new ApiKeyId(entityId.getId()))); @@ -126,6 +157,20 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService INCORRECT_API_KEY_ID + id); apiKeyDao.removeById(tenantId, apiKeyId); publishEvictEvent(new ApiKeyEvictEvent(apiKey.getValue())); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(apiKey.getId()).build()); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + ApiKey apiKey = findApiKeyById(tenantId, new ApiKeyId(id.getId())); + if (apiKey == null) { + if (force) { + return; + } else { + throw new DataValidationException("Unable to delete non-existent API key."); + } + } + deleteApiKey(tenantId, apiKey, force); } @Override @@ -144,6 +189,13 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService publishEvictEvent(new ApiKeyEvictEvent(value))); } + @Override + public PageData findApiKeysByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findApiKeysByTenantId [{}]", tenantId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + return apiKeyDao.findByTenantId(tenantId, pageLink); + } + @Override public ApiKey findApiKeyByValue(String value) { log.trace("Executing findApiKeyByValue [{}]", value); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java index e3e560725c..a83223d726 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.pat; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -22,6 +24,7 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.ApiKeyEntity; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -29,6 +32,10 @@ public interface ApiKeyRepository extends JpaRepository { ApiKeyEntity findByValue(String value); + Page findByTenantId(UUID tenantId, Pageable pageable); + + List findByTenantIdAndUserId(UUID tenantId, UUID userId); + @Transactional @Modifying @Query(value = """ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java index 61ca7bd142..69de36384e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java @@ -22,6 +22,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiKeyEntity; @@ -29,6 +31,7 @@ import org.thingsboard.server.dao.pat.ApiKeyDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -45,6 +48,16 @@ public class JpaApiKeyDao extends JpaAbstractDao implement return DaoUtil.getData(apiKeyRepository.findByValue(value)); } + @Override + public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiKeyRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public List findByTenantIdAndUserId(TenantId tenantId, UserId userId) { + return DaoUtil.convertDataList(apiKeyRepository.findByTenantIdAndUserId(tenantId.getId(), userId.getId())); + } + @Override public Set deleteByTenantId(TenantId tenantId) { return apiKeyRepository.deleteByTenantId(tenantId.getId()); From 47369026fb05552725f1fb572e364b6d227ea311 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 6 Mar 2026 14:25:16 +0200 Subject: [PATCH 11/71] CE: Fixed notification requests and RPC cleanup timeout on large datasets --- .../service/ttl/AbstractCleanUpService.java | 4 + .../ttl/NotificationsCleanUpService.java | 63 +++++++- .../service/ttl/rpc/RpcCleanUpService.java | 94 +++++++----- .../src/main/resources/thingsboard.yml | 4 +- .../ttl/NotificationsCleanUpServiceTest.java | 139 ++++++++++++++++++ .../ttl/rpc/RpcCleanUpServiceTest.java | 136 +++++++++++++++++ .../notification/NotificationRequestDao.java | 2 +- .../thingsboard/server/dao/rpc/RpcDao.java | 3 +- .../JpaNotificationRequestDao.java | 4 +- .../NotificationRequestRepository.java | 8 +- .../server/dao/sql/rpc/JpaRpcDao.java | 4 +- .../server/dao/sql/rpc/RpcRepository.java | 9 +- .../JpaNotificationRequestDaoTest.java | 133 +++++++++++++++++ .../server/dao/sql/rpc/JpaRpcDaoTest.java | 7 +- 14 files changed, 554 insertions(+), 56 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 5865d9e39d..596f5a8754 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -32,4 +32,8 @@ public abstract class AbstractCleanUpService { return partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); } + protected boolean isTenantPartitionMine(TenantId tenantId) { + return partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java index ddc95b74e9..5e327a72db 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java @@ -20,33 +20,41 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationRequestConfig; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.notification.NotificationRequestDao; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; +import java.time.Instant; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.dao.model.ModelConstants.NOTIFICATION_TABLE_NAME; +@Slf4j @Service @ConditionalOnExpression("${sql.ttl.notifications.enabled:true} && ${sql.ttl.notifications.ttl:0} > 0") -@Slf4j public class NotificationsCleanUpService extends AbstractCleanUpService { private final SqlPartitioningRepository partitioningRepository; private final NotificationRequestDao notificationRequestDao; + private final TenantService tenantService; @Value("${sql.ttl.notifications.ttl:2592000}") private long ttlInSec; @Value("${sql.notifications.partition_size:168}") private int partitionSizeInHours; + @Value("${sql.ttl.notifications.removal_batch_size:10000}") + private int removalBatchSize; public NotificationsCleanUpService(PartitionService partitionService, SqlPartitioningRepository partitioningRepository, - NotificationRequestDao notificationRequestDao) { + NotificationRequestDao notificationRequestDao, TenantService tenantService) { super(partitionService); this.partitioningRepository = partitioningRepository; this.notificationRequestDao = notificationRequestDao; + this.tenantService = tenantService; } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.notifications.checking_interval_ms:86400000})}", @@ -63,9 +71,56 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { if (lastRemovedNotificationTs > 0) { long gap = TimeUnit.MINUTES.toMillis(10); long requestExpTime = lastRemovedNotificationTs - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; - int removed = notificationRequestDao.removeAllByCreatedTimeBefore(requestExpTime); - log.info("Removed {} outdated notification requests older than {}", removed, requestExpTime); + cleanUpNotificationRequests(requestExpTime); + } + } + + private void cleanUpNotificationRequests(long expirationTime) { + log.info("Starting notification requests cleanup for records older than {}", Instant.ofEpochMilli(expirationTime)); + int totalRemoved = 0; + int tenantsProcessed = 0; + + // Clean up SYSADMIN's notification requests: + try { + totalRemoved += cleanUpByTenant(TenantId.SYS_TENANT_ID, expirationTime); + } catch (Exception e) { + log.warn("Failed to clean up notification requests for sysadmin {}", TenantId.SYS_TENANT_ID, e); + } + // Clean up notification requests for tenants + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; + } + int tenantRemoved = cleanUpByTenant(tenantId, expirationTime); + totalRemoved += tenantRemoved; + tenantsProcessed++; + if (tenantRemoved > 0) { + log.trace("Removed {} notification requests for tenant {}", tenantRemoved, tenantId); + } + } catch (Exception e) { + log.warn("Failed to clean up notification requests for tenant {}", tenantId, e); + } } + + log.info("Notification requests cleanup completed. Processed {} tenants, removed {} total records older than {}", tenantsProcessed, totalRemoved, Instant.ofEpochMilli(expirationTime)); + } + + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} notification requests in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index 9404a0ae72..a945bfd8a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -15,69 +15,87 @@ */ package org.thingsboard.server.service.ttl.rpc; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; -import java.util.Date; +import java.time.Instant; import java.util.Optional; import java.util.concurrent.TimeUnit; -@TbCoreComponent -@Service @Slf4j -@RequiredArgsConstructor -public class RpcCleanUpService { - @Value("${sql.ttl.rpc.enabled}") - private boolean ttlTaskExecutionEnabled; +@Service +@TbCoreComponent +@ConditionalOnExpression("${sql.ttl.rpc.enabled:true}") +public class RpcCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.rpc.removal_batch_size:10000}") + private int removalBatchSize; + private final RpcDao rpcDao; private final TenantService tenantService; - private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; - private final RpcDao rpcDao; + + public RpcCleanUpService(TenantService tenantService, PartitionService partitionService, TbTenantProfileCache tenantProfileCache, RpcDao rpcDao) { + super(partitionService); + this.tenantService = tenantService; + this.tenantProfileCache = tenantProfileCache; + this.rpcDao = rpcDao; + } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.rpc.checking_interval})}", fixedDelayString = "${sql.ttl.rpc.checking_interval}") public void cleanUp() { - if (ttlTaskExecutionEnabled) { - PageLink tenantsBatchRequest = new PageLink(10_000, 0); - PageData tenantsIds; - do { - tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); - for (TenantId tenantId : tenantsIds.getData()) { - if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { - continue; - } - - Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); - if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { - continue; - } - - long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); - long expirationTime = System.currentTimeMillis() - ttl; - - int totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime); - - if (totalRemoved > 0) { - log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, new Date(expirationTime)); - } + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; } - tenantsBatchRequest = tenantsBatchRequest.nextPageLink(); - } while (tenantsIds.hasNext()); + Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); + if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { + continue; + } + + long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); + long expirationTime = System.currentTimeMillis() - ttl; + + int totalRemoved = cleanUpByTenant(tenantId, expirationTime); + + if (totalRemoved > 0) { + log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, Instant.ofEpochMilli(expirationTime)); + } + } catch (Exception e) { + log.warn("Failed to clean up rpc by ttl for tenant {}", tenantId, e); + } } } + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} rpc in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; + } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 751d1c0e0b..8afdbb9ca9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -421,10 +421,11 @@ sql: edge_events_ttl: "${SQL_TTL_EDGE_EVENTS_TTL:2628000}" # Number of seconds. The current value corresponds to one month alarms: checking_interval: "${SQL_ALARMS_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours - removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # To delete outdated alarms not all at once but in batches + removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # Batch size for records removal rpc: enabled: "${SQL_TTL_RPC_ENABLED:true}" # Enable/disable TTL (Time To Live) for rpc call records checking_interval: "${SQL_RPC_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours + removal_batch_size: "${SQL_RPC_TTL_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal audit_logs: enabled: "${SQL_TTL_AUDIT_LOGS_ENABLED:true}" # Enable/disable TTL (Time To Live) for audit log records ttl: "${SQL_TTL_AUDIT_LOGS_SECS:0}" # Disabled by default. The accuracy of the cleanup depends on the sql.audit_logs.partition_size @@ -433,6 +434,7 @@ sql: enabled: "${SQL_TTL_NOTIFICATIONS_ENABLED:true}" # Enable/disable TTL (Time To Live) for notification center records ttl: "${SQL_TTL_NOTIFICATIONS_SECS:2592000}" # Default value - 30 days checking_interval_ms: "${SQL_TTL_NOTIFICATIONS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day + removal_batch_size: "${SQL_TTL_NOTIFICATIONS_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal relations: max_level: "${SQL_RELATIONS_MAX_LEVEL:50}" # This value has to be reasonably small to prevent infinite recursion as early as possible pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java new file mode 100644 index 0000000000..645eb600a4 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java @@ -0,0 +1,139 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.notification.NotificationRequestDao; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class NotificationsCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private SqlPartitioningRepository partitioningRepository; + @Mock + private NotificationRequestDao notificationRequestDao; + @Mock + private TenantService tenantService; + + private NotificationsCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new NotificationsCleanUpService(partitionService, partitioningRepository, notificationRequestDao, tenantService); + ReflectionTestUtils.setField(cleanUpService, "ttlInSec", 2592000L); + ReflectionTestUtils.setField(cleanUpService, "partitionSizeInHours", 168); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + + // Sysadmin: returns 3 (full batch), then 1 (partial) -> 2 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + // Tenant: returns 3, 3, 0 -> 3 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao, times(2)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, times(3)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + when(partitionService.resolve(any(), eq(TenantId.SYS_TENANT_ID), eq(TenantId.SYS_TENANT_ID))) + .thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + // Sysadmin: no records + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, never()).removeByTenantIdAndCreatedTimeBeforeBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testNoPartitionsDropped_skipsRequestCleanup() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(0L); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao, never()).removeByTenantIdAndCreatedTimeBeforeBatch(any(), anyLong(), anyInt()); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java new file mode 100644 index 0000000000..22cc1be1dd --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.rpc; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.rpc.RpcDao; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class RpcCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private RpcDao rpcDao; + @Mock + private TenantService tenantService; + @Mock + private TbTenantProfileCache tenantProfileCache; + + private RpcCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new RpcCleanUpService(tenantService, partitionService, tenantProfileCache, rpcDao); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 7); + + // Returns 3 (full batch), 3 (full batch), 1 (partial) -> 3 calls + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + + cleanUpService.cleanUp(); + + verify(rpcDao, times(3)).deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + setupTenantProfile(myTenant, 7); + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(rpcDao).deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testSkipsTenantWithZeroTtl() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 0); + + cleanUpService.cleanUp(); + + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(any(), anyLong(), anyInt()); + } + + private void setupTenant(TenantId tenantId, int rpcTtlDays) { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), eq(tenantId), eq(tenantId))).thenReturn(myPartition); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + setupTenantProfile(tenantId, rpcTtlDays); + } + + private void setupTenantProfile(TenantId tenantId, int rpcTtlDays) { + TenantProfile profile = new TenantProfile(); + TenantProfileData profileData = new TenantProfileData(); + DefaultTenantProfileConfiguration config = new DefaultTenantProfileConfiguration(); + config.setRpcTtlDays(rpcTtlDays); + profileData.setConfiguration(config); + profile.setProfileData(profileData); + when(tenantProfileCache.get(tenantId)).thenReturn(profile); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java index 96a86073d4..8e1408f4fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java @@ -50,7 +50,7 @@ public interface NotificationRequestDao extends Dao { boolean existsByTenantIdAndStatusAndTemplateId(TenantId tenantId, NotificationRequestStatus status, NotificationTemplateId templateId); - int removeAllByCreatedTimeBefore(long ts); + int removeByTenantIdAndCreatedTimeBeforeBatch(TenantId tenantId, long ts, int batchSize); NotificationRequestInfo findInfoById(TenantId tenantId, NotificationRequestId id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java index f88b672a34..37fe2950b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java @@ -24,12 +24,13 @@ import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.Dao; public interface RpcDao extends Dao { + PageData findAllByDeviceId(TenantId tenantId, DeviceId deviceId, PageLink pageLink); PageData findAllByDeviceIdAndStatus(TenantId tenantId, DeviceId deviceId, RpcStatus rpcStatus, PageLink pageLink); PageData findAllRpcByTenantId(TenantId tenantId, PageLink pageLink); - int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java index 9d32e91ca2..f4cc406a75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java @@ -98,8 +98,8 @@ public class JpaNotificationRequestDao extends JpaAbstractDao implements RpcDao, @Transactional @Override - public int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { - return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); + public int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize) { + return rpcRepository.deleteOutdatedRpcByTenantIdBatch(tenantId.getId(), expirationTime, batchSize); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java index 8a9489333f..7be106219b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java @@ -27,6 +27,7 @@ import org.thingsboard.server.dao.model.sql.RpcEntity; import java.util.UUID; public interface RpcRepository extends JpaRepository { + Page findAllByTenantIdAndDeviceId(UUID tenantId, UUID deviceId, Pageable pageable); Page findAllByTenantIdAndDeviceIdAndStatus(UUID tenantId, UUID deviceId, RpcStatus status, Pageable pageable); @@ -34,7 +35,11 @@ public interface RpcRepository extends JpaRepository { Page findAllByTenantId(UUID tenantId, Pageable pageable); @Modifying - @Query(value = "DELETE FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime", + @Query(value = "DELETE FROM rpc WHERE id IN " + + "(SELECT id FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime LIMIT :batchSize)", nativeQuery = true) - int deleteOutdatedRpcByTenantId(@Param("tenantId") UUID tenantId, @Param("expirationTime") Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(@Param("tenantId") UUID tenantId, + @Param("expirationTime") Long expirationTime, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java new file mode 100644 index 0000000000..5e2fe2308b --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.notification; + +import org.junit.After; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.NotificationRequest; +import org.thingsboard.server.common.data.notification.NotificationRequestStatus; +import org.thingsboard.server.dao.AbstractJpaDaoTest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JpaNotificationRequestDaoTest extends AbstractJpaDaoTest { + + @Autowired + JpaNotificationRequestDao notificationRequestDao; + + private final List createdRequests = new ArrayList<>(); + + @After + public void tearDown() { + for (NotificationRequest request : createdRequests) { + notificationRequestDao.removeById(request.getTenantId(), request.getId().getId()); + } + createdRequests.clear(); + } + + @Test + public void testBatchDeletion() { + TenantId sysTenantId = TenantId.SYS_TENANT_ID; + long now = System.currentTimeMillis(); + long oldTimestamp = now - TimeUnit.DAYS.toMillis(30); + + NotificationRequest oldRequest1 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest1); + + NotificationRequest oldRequest2 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest2); + + NotificationRequest freshRequest = createNotificationRequest(sysTenantId, now); + notificationRequestDao.save(sysTenantId, freshRequest); + + TenantId tenant2Id = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + NotificationRequest tenant2Request = createNotificationRequest(tenant2Id, oldTimestamp); + notificationRequestDao.save(tenant2Id, tenant2Request); + + int batchSize = 10_000; + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, oldTimestamp - 1, batchSize)).isEqualTo(0); + + long expirationTime = now - TimeUnit.DAYS.toMillis(15); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, expirationTime, batchSize)).isEqualTo(2); + + assertThat(notificationRequestDao.findById(sysTenantId, freshRequest.getId().getId())).isNotNull(); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2Id, now + 1, batchSize)).isEqualTo(1); + } + + @Test + public void testBatchDeletionWithSmallBatchSize() { + TenantId tenantId = TenantId.SYS_TENANT_ID; + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 10; i++) { + NotificationRequest request = createNotificationRequest(tenantId, oldTimestamp); + notificationRequestDao.save(tenantId, request); + } + + int batchSize = 3; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(1); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(0); + } + + @Test + public void testBatchDeletionIsolationBetweenTenants() { + TenantId tenant1 = TenantId.SYS_TENANT_ID; + TenantId tenant2 = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 5; i++) { + NotificationRequest request = createNotificationRequest(tenant1, oldTimestamp); + notificationRequestDao.save(tenant1, request); + } + + for (int i = 0; i < 3; i++) { + NotificationRequest request = createNotificationRequest(tenant2, oldTimestamp); + notificationRequestDao.save(tenant2, request); + } + + int batchSize = 10_000; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant1, expirationTime, batchSize)).isEqualTo(5); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2, expirationTime, batchSize)).isEqualTo(3); + } + + private NotificationRequest createNotificationRequest(TenantId tenantId, long createdTime) { + NotificationRequest request = new NotificationRequest(); + request.setId(new NotificationRequestId(UUID.randomUUID())); + request.setTenantId(tenantId); + request.setCreatedTime(createdTime); + request.setTargets(List.of(UUID.randomUUID())); + request.setStatus(NotificationRequestStatus.SENT); + createdRequests.add(request); + return request; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java index 1629922685..921339a92b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java @@ -51,9 +51,10 @@ public class JpaRpcDaoTest extends AbstractJpaDaoTest { rpc.setDeviceId(new DeviceId(UUID.randomUUID())); rpcDao.saveAndFlush(rpc.getTenantId(), rpc); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, 0L)).isEqualTo(0); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, Long.MAX_VALUE)).isEqualTo(2); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(tenantId, System.currentTimeMillis() + 1)).isEqualTo(1); + int batchSize = 10_000; + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, 0L, batchSize)).isEqualTo(0); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, Long.MAX_VALUE, batchSize)).isEqualTo(2); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, System.currentTimeMillis() + 1, batchSize)).isEqualTo(1); } } From e1a32a6e654581876e702b05993aa561d1b19673 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 6 Mar 2026 15:47:21 +0200 Subject: [PATCH 12/71] added env section description --- .../src/main/resources/thingsboard.yml | 43 ++++++++++++- msa/js-executor/config/default.yml | 64 +++++++++++-------- .../src/main/resources/tb-vc-executor.yml | 9 +++ .../src/main/resources/tb-coap-transport.yml | 16 ++++- .../src/main/resources/tb-http-transport.yml | 11 ++++ .../src/main/resources/tb-lwm2m-transport.yml | 11 ++++ .../src/main/resources/tb-mqtt-transport.yml | 12 ++++ .../src/main/resources/tb-snmp-transport.yml | 12 ++++ 8 files changed, 148 insertions(+), 30 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2131407cd2..1558d1cf74 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -15,6 +15,7 @@ # # Server common parameters +# Configures HTTP/HTTPS bind address, port, SSL, WebSocket, and REST API settings. server: # Server bind-address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -109,11 +110,13 @@ server: response_timeout: "${DEFAULT_RULE_ENGINE_RESPONSE_TIMEOUT:10000}" # Application info parameters +# Exposes application metadata such as version string injected at build time. app: # Application version version: "@project.version@" # Zookeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" @@ -133,6 +136,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cluster parameters +# Controls inter-node communication statistics within a ThingsBoard cluster. cluster: stats: # Enable/Disable the cluster statistics. Calculates the number of messages sent between cluster nodes based on each type @@ -141,11 +145,13 @@ cluster: print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" # Plugins configuration parameters +# Defines classpath scan packages used to discover and register ThingsBoard extension plugins. plugins: # Comma-separated package list used during classpath scanning for plugins scan_packages: "${PLUGINS_SCAN_PACKAGES:org.thingsboard.server.extensions,org.thingsboard.rule.engine}" # Security parameters +# Configures JWT tokens, user login, device claiming, OAuth2, and CA certificate trust store. security: # JWT Token parameters jwt: # Since 3.4.2 values are persisted in the database during installation or upgrade. On Install, the key will be generated randomly if no custom value set. You can change it later from Web UI under SYS_ADMIN @@ -184,6 +190,7 @@ security: password: "${SECURITY_JAVA_CACERTS_PASSWORD:changeit}" # Mail settings parameters +# Configures mail service OAuth2 token refresh and per-tenant sending rate limits. mail: oauth2: # Interval for checking refresh token expiration in seconds(by default, 1 day). @@ -192,6 +199,7 @@ mail: per_tenant_rate_limits: "${MAIL_PER_TENANT_RATE_LIMITS:}" # Usage statistics parameters +# Controls collection and reporting intervals for API usage stats at system, tenant, and customer levels. usage: stats: report: @@ -215,6 +223,7 @@ usage: report_interval: "${DEVICES_STATS_REPORT_INTERVAL:60}" # UI settings parameters +# Configures dashboard data limits and base URL for UI help assets. ui: # Dashboard parameters dashboard: @@ -226,6 +235,7 @@ ui: base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.4}" # Database telemetry parameters +# Selects the storage backend (SQL, Cassandra, or TimescaleDB) for time-series and latest telemetry data. database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by a single API call to fetch telemetry records ts: @@ -234,6 +244,7 @@ database: type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) # Cassandra driver configuration parameters +# Configures cluster name, keyspace, SSL, credentials, socket, and query settings for the Cassandra driver. cassandra: # Thingsboard cluster name cluster_name: "${CASSANDRA_CLUSTER_NAME:Thingsboard Cluster}" @@ -352,6 +363,7 @@ cassandra: print_tenant_names: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_PRINT_TENANT_NAMES:false}" # SQL configuration parameters +# Tunes batch sizes, delays, thread counts, TTL, and partitioning for SQL-backed persistence of telemetry, events, and audit logs. sql: # Specify batch size for persisting attribute updates attributes: @@ -453,6 +465,7 @@ sql: query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls # Actor system parameters +# Configures thread pools, timeouts, and behavior for the internal actor system processing devices, rules, and RPCs. actors: system: throughput: "${ACTORS_SYSTEM_THROUGHPUT:5}" # Number of messages the actor system will process per actor before switching to processing of messages for the next actor @@ -553,6 +566,8 @@ actors: # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" +# Debug settings parameters +# Configures the global default duration for debug mode used by rule chains and calculated fields. debug: settings: # Default duration (in minutes) for debug mode. Min value is 1 minute. Tenant profile settings override this one. @@ -560,6 +575,7 @@ debug: default_duration: "${DEBUG_SETTINGS_DEFAULT_DURATION_MINUTES:15}" # Cache settings parameters +# Configures cache type (Caffeine or Redis), pool size, and per-entity TTL and max-size limits for all caches. cache: # caffeine or redis(7.2 - latest compatible version) type: "${CACHE_TYPE:caffeine}" @@ -719,6 +735,7 @@ cache: spring.data.redis.repositories.enabled: false # Disable this because it is not required. # Redis/Valkey configuration parameters +# Configures standalone, cluster, or sentinel Redis connection, SSL, and connection pool settings used for caching. redis: # standalone or cluster or sentinel connection: @@ -802,11 +819,13 @@ redis: # Update version parameters +# Controls whether the platform periodically checks for new ThingsBoard releases. updates: # Enable/disable checks for the new version enabled: "${UPDATES_ENABLED:true}" # Spring CORS configuration parameters +# Defines allowed origins, methods, headers, and credentials for cross-origin REST API requests. spring.mvc.cors: mappings: # Intercept path @@ -823,6 +842,7 @@ spring.mvc.cors: allow-credentials: "true" # General spring parameters +# Miscellaneous Spring Boot settings for circular references, Freemarker, MVC, multipart, and JPA dialect. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. spring.freemarker.checkTemplateLocation: "false" # spring freemarker configuration spring.mvc.async.request-timeout: "${SPRING_MVC_ASYNC_REQUEST_TIMEOUT:30000}" # The default timeout for asynchronous requests in milliseconds @@ -841,6 +861,7 @@ spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PR spring.jpa.properties.hibernate.dialect: "${SPRING_JPA_DIALECT:org.thingsboard.server.dao.ThingsboardPostgreSQLDialect}" # we use custom dialect that contains ilike(arg1, arg2) function (is interpreted to postgres ILIKE operator) # SQL DAO Configuration parameters +# Configures Spring JPA, Hibernate DDL, and HikariCP datasource connections for the primary and events databases. spring: data: jpa: @@ -891,6 +912,7 @@ spring: registerMbeans: "${SPRING_EVENTS_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # Audit log parameters +# Configures audit logging levels per entity type and optional forwarding to an external sink such as Elasticsearch. audit-log: # Enable/disable audit log functionality. enabled: "${AUDIT_LOG_ENABLED:true}" @@ -933,6 +955,7 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" # Password used to access external sink system # Device state parameters +# Configures device inactivity detection, state persistence strategy, and rate limits for state rule nodes. state: # Device inactivity timeout is a global configuration parameter that defines when the device will be marked as "inactive" by the server. # The parameter value is in seconds. A user can overwrite this parameter for an individual device by setting the “inactivityTimeout” server-side attribute (NOTE: expects value in milliseconds). @@ -963,7 +986,8 @@ state: # Refill is set to be greedy. Please refer to Bucket4j library documentation for more details. rateLimit: "${DEVICE_STATE_NODE_RATE_LIMIT_CONFIGURATION:1:1,30:60,60:3600}" -# Tbel parameters +# TBEL parameters +# Configures the ThingsBoard Expression Language (TBEL) engine: limits, timeouts, thread pool, and blacklisting. tbel: # Enable/Disable TBEL feature. enabled: "${TBEL_ENABLED:true}" @@ -992,6 +1016,7 @@ tbel: print_interval_ms: "${TB_TBEL_STATS_PRINT_INTERVAL_MS:10000}" # JS parameters +# Configures the JavaScript execution engine (local Nashorn or remote Node.js): limits, sandboxing, and thread pools. js: # local (Nashorn Engine, deprecated) OR remote JS-Executors (NodeJS) evaluator: "${JS_EVALUATOR:local}" @@ -1039,6 +1064,7 @@ js: print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" # Transport configuration parameters +# Configures session management, rate limits, and protocol-specific settings for HTTP, MQTT, CoAP, LwM2M, and SNMP transports. transport: sessions: # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device. @@ -1349,6 +1375,7 @@ transport: fetch_frequency: "${TB_GATEWAY_DASHBOARD_SYNC_FETCH_FREQUENCY:24}" # CoAP server parameters +# Configures the standalone CoAP server bind address, port, DTLS encryption, and credential settings. coap: server: # Enable/disable coap server. @@ -1433,6 +1460,7 @@ coap: dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Device connectivity parameters +# Specifies the hosts, ports, and credentials exposed to the UI for generating device connection check commands. device: connectivity: http: @@ -1486,6 +1514,7 @@ device: image_version: "${DEVICE_CONNECTIVITY_GATEWAY_IMAGE_VERSION:3.8-stable}" # Edges parameters +# Controls Edge instance gRPC communication, event storage, state persistence, and statistics reporting. edges: # Enable/disable Edge instance enabled: "${EDGES_ENABLED:true}" @@ -1547,6 +1576,7 @@ edges: report-interval-millis: "${EDGES_STATS_REPORT_INTERVAL_MS:600000}" # Spring doc common parameters +# Enables or disables the OpenAPI/Swagger documentation endpoint and sets the default media type. springdoc: # If false swagger API docs will be unavailable api-docs.enabled: "${SWAGGER_ENABLED:true}" @@ -1554,6 +1584,7 @@ springdoc: default-produces-media-type: "${SWAGGER_DEFAULT_PRODUCES_MEDIA_TYPE:application/json}" # Swagger common parameters +# Configures Swagger UI metadata: title, description, contact, license, version, and API path patterns. swagger: # General swagger match pattern of swagger UI links api_path: "${SWAGGER_API_PATH:/api/**}" @@ -1585,6 +1616,7 @@ swagger: doc_expansion: "${SWAGGER_DOC_EXPANSION:list}" # Queue configuration parameters +# Configures the message queue backend (in-memory or Kafka) and all topic/partition/consumer settings for each service. queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -1998,12 +2030,14 @@ queue: processing_interval: "${TB_QUEUE_TASKS_STATS_PROCESSING_INTERVAL_MS:1000}" # Event configuration parameters +# Controls limits on debug event content size stored for rule chain and rule node execution. event: debug: # Maximum number of symbols per debug event. The event content will be truncated if needed max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" # General service parameters +# Sets the deployment type (monolith or microservice) and assigns tenant profiles to specific Rule Engine instances. service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) @@ -2017,6 +2051,7 @@ service: executor_thread_pool_size: "${TB_RULE_ENGINE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" # Metrics parameters +# Enables actuator metrics endpoint and configures system info (CPU, memory) persistence frequency and TTL. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" @@ -2030,6 +2065,7 @@ metrics: ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" # Version control parameters +# Configures thread pools and Git repository folder for entity version control (export/import) operations. vc: # Pool size for handling export tasks thread_pool_size: "${TB_VC_POOL_SIZE:6}" @@ -2040,6 +2076,7 @@ vc: repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" # Notification system parameters +# Configures thread pool size and deduplication intervals for the notification rule processing engine. notification_system: # Specify thread pool size for Notification System processing notification rules and notification sending. Recommend value <= 10 thread_pool_size: "${TB_NOTIFICATION_SYSTEM_THREAD_POOL_SIZE:10}" @@ -2048,6 +2085,7 @@ notification_system: deduplication_durations: "${TB_NOTIFICATION_RULES_DEDUPLICATION_DURATIONS:NEW_PLATFORM_VERSION:0;RATE_LIMITS:14400000;}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure and health indicator settings. management: endpoints: web: @@ -2060,6 +2098,7 @@ management: enabled: "false" # Mobile application settings for Thingsboard mobile application +# Specifies domain name and store links used for the ThingsBoard Live mobile application QR code and deep links. mobileApp: # Server domain name for Thingsboard Live mobile application domain: "${TB_MOBILE_APP_DOMAIN:demo.thingsboard.io}" @@ -2068,6 +2107,8 @@ mobileApp: # Link to App Store for Thingsboard Live mobile application appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}" +# MQTT client settings +# Configures MQTT client retransmission behavior including max attempts, initial delay, and jitter factor. mqtt: # MQTT client configuration parameters client: diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index be49ece5a6..adba4a3409 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,46 +14,56 @@ # limitations under the License. # -queue_type: "kafka" -request_topic: "js_eval.requests" -queue_prefix: "" -http_port: "8888" # /livenessProbe +# Queue configuration +# Defines the messaging queue backend used to dispatch JS evaluation requests to the executor and receive results. +queue_type: "kafka" # Queue type used for JS evaluation requests: kafka or in-memory +request_topic: "js_eval.requests" # Kafka topic name for incoming JS evaluation requests +queue_prefix: "" # Optional prefix applied to all queue/topic names; useful for environment isolation +http_port: "8888" # HTTP port exposing the /livenessProbe health check endpoint +# JavaScript execution configuration +# Controls response polling behavior and output size limits for individual JS script executions. js: - response_poll_interval: "25" - max_result_size: "300000" + response_poll_interval: "25" # Interval in milliseconds to poll for JS execution responses + max_result_size: "300000" # Maximum allowed size in bytes of a JS execution result +# Kafka configuration +# Connection, producer, and topic settings for the Kafka message broker used as the JS evaluation queue. kafka: bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" - replication_factor: "1" + replication_factor: "1" # Number of replicas for each Kafka topic partition acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge - batch_size: "128" # for producer - linger_ms: "5" # for producer + batch_size: "128" # Maximum number of bytes the producer batches per partition before sending + linger_ms: "5" # Time in milliseconds the producer waits before sending a batch, allowing more messages to accumulate partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency - requestTimeout: "30000" # The default value in kafkajs is: 30000 - connectionTimeout: "1000" # The default value in kafkajs is: 1000 - compression: "none" # gzip or uncompressed - topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" - use_confluent_cloud: false - client_id: "kafkajs" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh + requestTimeout: "30000" # Time in milliseconds to wait for a response to a Kafka request before timing out + connectionTimeout: "1000" # Time in milliseconds to wait when establishing a connection to a Kafka broker + compression: "none" # Message compression codec for the producer: none, gzip + topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" # Semicolon-separated Kafka topic configuration properties applied on topic creation + use_confluent_cloud: false # Set to true to enable Confluent Cloud-specific configuration (SSL + SASL) + client_id: "kafkajs" # Kafka client identifier; inject the pod name to easily identify the client via kafka-consumer-groups.sh ssl: - enabled: false + enabled: false # Enable SSL/TLS for Kafka broker connections confluent: sasl: - mechanism: "PLAIN" + mechanism: "PLAIN" # SASL mechanism for Confluent Cloud authentication: PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512 +# Logging configuration +# Controls log verbosity, output directory, and log file naming pattern for the JS executor service. logger: - level: "info" - path: "logs" - filename: "tb-js-executor-%DATE%.log" + level: "info" # Log level: trace, debug, info, warn, or error + path: "logs" # Directory path where log files are written + filename: "tb-js-executor-%DATE%.log" # Log file name pattern; %DATE% is replaced with the current date +# JavaScript execution and monitoring configuration +# Sandboxing, script caching, and observability settings for the JS script runtime. script: - use_sandbox: "true" - memory_usage_trace_frequency: "1000" - script_body_trace_frequency: "10000" - stat_print_frequency: "10000" - max_active_scripts: "1000" - slow_query_log_ms: "5.000000" #millis - slow_query_log_body: "false" + use_sandbox: "true" # Run scripts inside an isolated sandbox to prevent access to Node.js internals + memory_usage_trace_frequency: "1000" # Log memory usage every N script executions + script_body_trace_frequency: "10000" # Log the script body every N script executions + stat_print_frequency: "10000" # Print execution statistics every N script executions + max_active_scripts: "1000" # Maximum number of compiled scripts kept in the cache + slow_query_log_ms: "5.000000" # Execution time threshold in milliseconds above which a script is considered slow and logged + slow_query_log_body: "false" # Log the script body when execution time exceeds slow_query_log_ms threshold diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 56e15d4b4d..b52ad1ad89 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -15,11 +15,13 @@ # # Spring common parameters +# Controls Spring Boot startup mode, web environment type, and bean wiring behaviour. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port used for actuator and metrics endpoints. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -46,6 +48,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Queue configuration parameters +# Defines the message queue type (Kafka or in-memory) and all related producer, consumer, and topic settings. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -196,6 +199,7 @@ queue: msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" # Version control parameters +# Configures the Git-based version control executor, including thread pools and local repository storage. vc: # Pool size for handling export tasks thread_pool_size: "${TB_VC_POOL_SIZE:6}" @@ -206,6 +210,7 @@ vc: repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" # Usage statistics parameters +# Controls collection and reporting of API usage statistics at system, tenant, and customer levels. usage: stats: report: @@ -221,6 +226,7 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables actuator-based metrics collection and configures histogram/timer percentile exports. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" @@ -229,6 +235,7 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including the metrics and info endpoints. management: endpoints: web: @@ -237,12 +244,14 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Service common properties +# Identifies this microservice instance by type and optional unique ID for cluster coordination. service: type: "${TB_SERVICE_TYPE:tb-vc-executor}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Notification system parameters +# Defines deduplication rules for notification triggers to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 376075a0a4..e05ecd18b0 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -15,11 +15,13 @@ # # Spring common parameters +# Controls core Spring Boot application settings such as web environment and application type. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the HTTP server bind address and port for the actuator/metrics web endpoint. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -27,6 +29,7 @@ server: port: "${HTTP_BIND_PORT:8083}" # Zookeeper connection parameters. Used for service discovery. +# Defines how this service registers itself and discovers other nodes via Apache ZooKeeper. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the caching backend and per-entity cache settings used by the transport service. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Defines connection mode, authentication, SSL, and connection pool settings for Redis or Valkey. redis: connection: # standalone or cluster or sentinel @@ -90,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -172,6 +177,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # CoAP server parameters +# Configures the CoAP and CoAP/DTLS server endpoints, including bind addresses, timeouts, and TLS credentials. coap: # CoAP bind-address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" @@ -253,6 +259,7 @@ coap: dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Queue configuration parameters +# Defines the message queue backend (Kafka) and all topic, partition, and consumer settings for inter-service communication. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -406,12 +413,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common properties +# Identifies this service instance within the ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage metrics at system, tenant, and customer levels. usage: stats: report: @@ -427,11 +436,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables Micrometer/Actuator metrics collection and configures the exposed management endpoints. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure for health checks and metrics scraping. management: endpoints: web: @@ -440,6 +451,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for platform notifications to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 1f6a251324..0ce5173ed4 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -15,6 +15,7 @@ # # Server common properties +# Configures the HTTP server bind address, port, SSL/TLS settings, and HTTP/2 support. server: # Server bind address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -54,6 +55,7 @@ server: enabled: "${HTTP2_ENABLED:true}" # Spring common parameters +# Controls Spring Boot framework-level settings such as circular references and multipart upload limits. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. spring.servlet.multipart.max-file-size: "${SPRING_SERVLET_MULTIPART_MAX_FILE_SIZE:50MB}" # Total file size cannot exceed 50MB when configuring file uploads spring.servlet.multipart.max-request-size: "${SPRING_SERVLET_MULTIPART_MAX_REQUEST_SIZE:50MB}" # Total request size for a multipart/form-data cannot exceed 50MB @@ -78,6 +80,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Defines the cache backend type and per-entity cache configuration such as TTL and maximum size. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -87,6 +90,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Configures connection mode (standalone, cluster, sentinel), credentials, SSL, and connection pool settings. redis: # standalone or cluster or sentinel connection: @@ -166,6 +170,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # HTTP server parameters +# Configures HTTP transport request timeouts, payload limits, session management, JSON processing, logging, and statistics. transport: http: # HTTP request processing timeout in milliseconds @@ -203,6 +208,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration parameters +# Configures the messaging queue backend (Kafka), topic definitions, partitioning, and per-service consumer settings. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka) . @@ -355,12 +361,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # General service parameters +# Defines the service type and unique identifier used for service discovery and cluster coordination. service: type: "${TB_SERVICE_TYPE:tb-transport}" # type of service # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at the system, tenant, and optional customer level. usage: stats: report: @@ -376,11 +384,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables actuator metrics collection and exposure for monitoring integrations such as Prometheus. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator management endpoints, including which endpoints are exposed over HTTP. management: endpoints: web: @@ -389,6 +399,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures notification rules, including deduplication windows to prevent repeated alerts for the same trigger. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index b4d8d6c7eb..2bc1e6667d 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -15,11 +15,13 @@ # # Spring common parameters +# Controls Spring Boot web environment and application type settings used for optional metrics exposure. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# HTTP server bind address and port, effective only when the web environment is enabled. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -46,6 +48,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the cache backend type and per-entity cache specifications such as TTL and maximum size. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +58,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Connection, authentication, SSL, and connection pool settings for the Redis or Valkey cache backend. redis: connection: # standalone or cluster or sentinel @@ -133,6 +137,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # LWM2M server parameters +# Session, JSON processing, RPC, logging, and LwM2M-specific transport settings including DTLS, bootstrap, and OTA. transport: sessions: # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device. @@ -303,6 +308,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration properties +# Defines the message queue backend (Kafka) and per-topic settings for transport, core, rule-engine, and JS evaluation. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -456,12 +462,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common parameters +# Identifies the service type and its unique instance ID within a ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at system, tenant, and optionally customer level. usage: stats: report: @@ -477,11 +485,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables Actuator-based metrics collection (e.g., Prometheus endpoint). metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, e.g. for Prometheus metrics scraping. management: endpoints: web: @@ -490,6 +500,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for notification triggers to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 0b96c57c13..9298b06d74 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -15,11 +15,13 @@ # # Spring common parameters +# Controls Spring Boot application type, web environment, and bean dependency settings. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port for actuator/metrics endpoints. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -27,6 +29,7 @@ server: port: "${HTTP_BIND_PORT:8083}" # Zookeeper connection parameters. Used for service discovery. +# Defines connection, session, retry settings and the node directory path for Zookeeper-based cluster coordination. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the cache backend type (Caffeine or Redis) and per-cache TTL and size limits. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Covers connection mode (standalone, cluster, sentinel), SSL, credentials, and connection pool tuning. redis: # standalone or cluster or sentinel connection: @@ -134,6 +139,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # MQTT server parameters +# Configures the MQTT transport layer including bind address/port, SSL, Netty tuning, session management, rate limits, and message processing options. transport: mqtt: # MQTT bind-address @@ -236,6 +242,7 @@ transport: ip_block_timeout: "${TB_TRANSPORT_IP_BLOCK_TIMEOUT:60000}" # Queue configuration parameters +# Defines the messaging queue backend (Kafka), topic names, partitioning, and poll/processing settings for all platform microservices. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -389,12 +396,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common properties +# Identifies this service instance within the cluster by type and optional unique ID. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at system, tenant, and customer levels with configurable intervals. usage: stats: report: @@ -410,11 +419,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables Micrometer actuator metrics collection for monitoring this service. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including which endpoints are accessible over HTTP. management: endpoints: web: @@ -423,6 +434,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for notification triggers to prevent repeated alerts within a defined time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 71524ba47e..a3491170da 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -15,11 +15,13 @@ # # Spring common parameters +# Controls Spring Boot application type and web environment settings for the SNMP transport service. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port (used when web environment is enabled). server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -27,6 +29,7 @@ server: port: "${HTTP_BIND_PORT:8083}" # Zookeeper connection parameters. Used for service discovery. +# Defines ZooKeeper connectivity, timeouts, and cluster node registration settings for service coordination. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Specifies the caching backend (Caffeine or Redis) and per-entity cache TTL and size limits. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Covers standalone, cluster, and sentinel Redis connection modes, SSL, authentication, and connection pool tuning. redis: connection: # standalone or cluster or sentinel @@ -133,6 +138,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # Snmp server parameters +# Configures the SNMP transport protocol, bind port, PDU limits, session management, JSON handling, logging, and statistics. transport: snmp: # Enable/disable SNMP transport protocol @@ -184,6 +190,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration parameters +# Defines the message queue backend (Kafka) and topic/consumer settings for transport, core, rule-engine, and JS evaluation. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -344,12 +351,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common parameters +# Identifies this service instance by type and unique ID within a ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls collection and reporting intervals for API usage statistics at system, tenant, and customer levels. usage: stats: report: @@ -365,11 +374,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables actuator-based metrics collection (e.g., Prometheus) for the SNMP transport service. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including which management endpoints are accessible over HTTP. management: endpoints: web: @@ -378,6 +389,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Defines deduplication rules for notification triggers to prevent duplicate alerts within configured time windows. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' From cda0a96f8d09e8a2b8db635df34c00cbdce6107c Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 6 Mar 2026 16:37:04 +0200 Subject: [PATCH 13/71] WebSocket API key authentication --- .../config/ApiKeyHandshakeInterceptor.java | 69 +++++++ .../server/config/WebSocketConfiguration.java | 5 +- .../controller/plugin/TbWebSocketHandler.java | 34 ++- .../pat/ApiKeyAuthenticationProvider.java | 2 +- .../server/service/ws/AuthCmd.java | 3 + .../controller/AbstractControllerTest.java | 18 +- .../controller/ApiKeyWebSocketApiTest.java | 67 ++++++ .../controller/TbTestWebSocketClient.java | 16 +- ...cketApiTest.java => WebSocketApiTest.java} | 195 +++++++++--------- .../plugin/TbWebSocketHandlerTest.java | 106 ++++++++++ 10 files changed, 406 insertions(+), 109 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/ApiKeyHandshakeInterceptor.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java rename application/src/test/java/org/thingsboard/server/controller/{WebsocketApiTest.java => WebSocketApiTest.java} (84%) diff --git a/application/src/main/java/org/thingsboard/server/config/ApiKeyHandshakeInterceptor.java b/application/src/main/java/org/thingsboard/server/config/ApiKeyHandshakeInterceptor.java new file mode 100644 index 0000000000..d6d550b2c0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/ApiKeyHandshakeInterceptor.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.WebSocketHandler; +import org.springframework.web.socket.server.HandshakeInterceptor; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.auth.pat.ApiKeyAuthenticationProvider; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.Map; + +@Slf4j +@Component +@TbCoreComponent +@RequiredArgsConstructor +public class ApiKeyHandshakeInterceptor implements HandshakeInterceptor { + + public static final String API_KEY_HEADER = "X-API-Key"; + public static final String API_KEY_SECURITY_CTX_ATTR = "apiKeySecurityCtx"; + + private final ApiKeyAuthenticationProvider apiKeyAuthenticationProvider; + + @Override + public boolean beforeHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Map attributes) { + String apiKey = request.getHeaders().getFirst(API_KEY_HEADER); + if (apiKey != null) { + if (apiKey.isEmpty()) { + log.debug("Empty API key provided during WS handshake"); + response.setStatusCode(HttpStatus.UNAUTHORIZED); + return false; + } + try { + SecurityUser securityUser = apiKeyAuthenticationProvider.authenticate(apiKey); + attributes.put(API_KEY_SECURITY_CTX_ATTR, securityUser); + } catch (Exception e) { + log.debug("API key authentication failed during WS handshake: {}", e.getMessage()); + response.setStatusCode(HttpStatus.UNAUTHORIZED); + return false; + } + } + return true; + } + + @Override + public void afterHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Exception exception) { + // no-op + } + +} diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 38f37cf35d..f5be3afa41 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -40,6 +40,7 @@ public class WebSocketConfiguration implements WebSocketConfigurer { private static final String WS_API_MAPPING = "/api/ws/**"; private final WebSocketHandler wsHandler; + private final ApiKeyHandshakeInterceptor apiKeyHandshakeInterceptor; @Value("${server.ws.max_text_message_buffer_size:32768}") private int maxTextMessageBufferSize; @@ -60,7 +61,9 @@ public class WebSocketConfiguration implements WebSocketConfigurer { log.error("TbWebSocketHandler expected but [{}] provided", wsHandler); throw new RuntimeException("TbWebSocketHandler expected but " + wsHandler + " provided"); } - registry.addHandler(wsHandler, WS_API_MAPPING).setAllowedOriginPatterns("*"); + registry.addHandler(wsHandler, WS_API_MAPPING) + .addInterceptors(apiKeyHandshakeInterceptor) + .setAllowedOriginPatterns("*"); } } diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 73315be73f..5c24c1e6c1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -48,10 +48,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.config.ApiKeyHandshakeInterceptor; import org.thingsboard.server.config.WebSocketConfiguration; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; +import org.thingsboard.server.service.security.auth.pat.ApiKeyAuthenticationProvider; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -100,6 +102,8 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke private RateLimitService rateLimitService; @Autowired private JwtAuthenticationProvider authenticationProvider; + @Autowired + private ApiKeyAuthenticationProvider apiKeyAuthenticationProvider; @Value("${server.ws.send_timeout:5000}") private long sendTimeout; @@ -194,7 +198,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke log.trace("{} Authenticating session", sessionRef); SecurityUser securityCtx; try { - securityCtx = authenticationProvider.authenticate(authCmd.getToken()); + if (StringUtils.isNotEmpty(authCmd.getApiKey())) { + securityCtx = apiKeyAuthenticationProvider.authenticate(authCmd.getApiKey()); + } else { + securityCtx = authenticationProvider.authenticate(authCmd.getToken()); + } } catch (Exception e) { close(sessionRef, CloseStatus.BAD_DATA.withReason(e.getMessage())); return; @@ -328,9 +336,17 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke } SecurityUser securityCtx = null; - String token = StringUtils.substringAfter(session.getUri().getQuery(), "token="); - if (StringUtils.isNotEmpty(token)) { - securityCtx = authenticationProvider.authenticate(token); + Object apiKeyCtx = session.getAttributes().get(ApiKeyHandshakeInterceptor.API_KEY_SECURITY_CTX_ATTR); + if (apiKeyCtx instanceof SecurityUser) { + securityCtx = (SecurityUser) apiKeyCtx; + } else { + String query = session.getUri().getQuery(); + if (query != null) { + String token = extractQueryParam(query, "token"); + if (StringUtils.isNotEmpty(token)) { + securityCtx = authenticationProvider.authenticate(token); + } + } } return WebSocketSessionRef.builder() .sessionId(UUID.randomUUID().toString()) @@ -341,6 +357,15 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke .build(); } + private String extractQueryParam(String query, String paramName) { + for (String param : query.split("&")) { + if (param.startsWith(paramName + "=")) { + return param.substring(paramName.length() + 1); + } + } + return null; + } + private SessionMetaData getSessionMd(String internalSessionId) { SessionMetaData sessionMd = internalSessionMap.get(internalSessionId); if (sessionMd == null) { @@ -482,6 +507,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke } } } + } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java index 162a780bbb..46af0120da 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java @@ -46,7 +46,7 @@ public class ApiKeyAuthenticationProvider extends AbstractAuthenticationProvider return new ApiKeyAuthenticationToken(securityUser); } - private SecurityUser authenticate(String key) { + public SecurityUser authenticate(String key) { if (StringUtils.isEmpty(key)) { throw new BadCredentialsException("Empty API key"); } diff --git a/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java index e139498d04..a2dc87e4af 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java @@ -23,11 +23,14 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @AllArgsConstructor public class AuthCmd implements WsCmd { + private int cmdId; private String token; + private String apiKey; @Override public WsCmdType getType() { return WsCmdType.AUTH; } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index c25dafa5e6..fa43f83ff2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -32,6 +32,7 @@ import org.springframework.web.socket.config.annotation.EnableWebSocket; import java.net.URI; import java.net.URISyntaxException; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -86,12 +87,12 @@ public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { } @Before - public void beforeWsTest() throws Exception { + public void beforeWsTest() { // placeholder } @After - public void afterWsTest() throws Exception { + public void afterWsTest() { if (wsClient != null) { wsClient.close(); } @@ -113,4 +114,17 @@ public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { return wsClient; } + protected TbTestWebSocketClient buildAndConnectWebSocketClientWithApiKey(String apiKey) throws URISyntaxException, InterruptedException { + TbTestWebSocketClient wsClient = new TbTestWebSocketClient(new URI(WS_URL + wsPort + "/api/ws")); + assertThat(wsClient.connectBlocking(TIMEOUT, TimeUnit.SECONDS)).isTrue(); + wsClient.authenticateWithApiKey(apiKey); + return wsClient; + } + + protected TbTestWebSocketClient buildAndConnectWebSocketClientWithApiKeyHeader(String apiKey) throws URISyntaxException, InterruptedException { + TbTestWebSocketClient wsClient = new TbTestWebSocketClient(new URI(WS_URL + wsPort + "/api/ws"), Map.of("X-API-Key", apiKey)); + assertThat(wsClient.connectBlocking(TIMEOUT, TimeUnit.SECONDS)).isTrue(); + return wsClient; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java new file mode 100644 index 0000000000..b4e96beee2 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.common.data.pat.ApiKeyInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.net.URISyntaxException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +@DaoSqlTest +public class ApiKeyWebSocketApiTest extends WebSocketApiTest { + + private ApiKey apiKey; + + @Before + public void setUpApiKey() { + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setDescription("WS test API key"); + apiKeyInfo.setEnabled(true); + apiKeyInfo.setUserId(tenantAdminUserId); + apiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + } + + @After + public void tearDownApiKey() throws Exception { + loginTenantAdmin(); + doDelete("/api/apiKey/" + apiKey.getId()).andExpect(status().isOk()); + } + + @Override + protected TbTestWebSocketClient buildAndConnectWebSocketClient() throws URISyntaxException, InterruptedException { + return buildAndConnectWebSocketClientWithApiKey(apiKey.getValue()); + } + + @Test + public void testApiKeyHeaderAuth() throws Exception { + TbTestWebSocketClient client = buildAndConnectWebSocketClientWithApiKeyHeader(apiKey.getValue()); + try { + assertThat(client.isOpen()).isTrue(); + } finally { + client.close(); + } + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java index 26705ddacb..35536eaecc 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java @@ -39,12 +39,12 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityHistoryCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; -import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.net.URI; import java.nio.channels.NotYetConnectedException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -62,6 +62,10 @@ public class TbTestWebSocketClient extends WebSocketClient { super(serverUri); } + public TbTestWebSocketClient(URI serverUri, Map httpHeaders) { + super(serverUri, httpHeaders); + } + @Override public void onOpen(ServerHandshake serverHandshake) { @@ -69,7 +73,13 @@ public class TbTestWebSocketClient extends WebSocketClient { public void authenticate(String token) { WsCommandsWrapper cmdsWrapper = new WsCommandsWrapper(); - cmdsWrapper.setAuthCmd(new AuthCmd(1, token)); + cmdsWrapper.setAuthCmd(new AuthCmd(1, token, null)); + send(JacksonUtil.toString(cmdsWrapper)); + } + + public void authenticateWithApiKey(String apiKey) { + WsCommandsWrapper cmdsWrapper = new WsCommandsWrapper(); + cmdsWrapper.setAuthCmd(new AuthCmd(1, null, apiKey)); send(JacksonUtil.toString(cmdsWrapper)); } @@ -275,7 +285,7 @@ public class TbTestWebSocketClient extends WebSocketClient { public JsonNode sendTimeseriesCmd(EntityId entityId, String scope) { log.warn("sendTimeseriesCmd entityId: {}, scope: {}", entityId, scope); - TimeseriesSubscriptionCmd cmd = new TimeseriesSubscriptionCmd(0, 0, 0, 10, null); + TimeseriesSubscriptionCmd cmd = new TimeseriesSubscriptionCmd(0, 0, 0, 10, null); cmd.setEntityId(entityId.getId().toString()); cmd.setEntityType(entityId.getEntityType().toString()); cmd.setCmdId(1); diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java similarity index 84% rename from application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java rename to application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java index 87ba0ec3e8..b7d03b5bea 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java @@ -71,13 +71,11 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; -import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -91,7 +89,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. "server.ws.alarms_per_alarm_status_subscription_cache_size=5", "server.ws.dynamic_page_link.refresh_interval=15" }) -public class WebsocketApiTest extends AbstractControllerTest { +public class WebSocketApiTest extends AbstractControllerTest { + @Autowired private TelemetrySubscriptionService tsService; @@ -128,8 +127,8 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertEquals(0, pageData.getData().get(0).getTimeseries().get("temperature").length); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertEquals(0, pageData.getData().getFirst().getTimeseries().get("temperature").length); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); TsKvEntry dataPoint2 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(2), new LongDataEntry("temperature", 42L)); @@ -144,8 +143,8 @@ public class WebsocketApiTest extends AbstractControllerTest { List dataList = update.getUpdate(); Assert.assertNotNull(dataList); Assert.assertEquals(1, dataList.size()); - Assert.assertEquals(device.getId(), dataList.get(0).getEntityId()); - TsValue[] tsArray = dataList.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), dataList.getFirst().getEntityId()); + TsValue[] tsArray = dataList.getFirst().getTimeseries().get("temperature"); Assert.assertEquals(3, tsArray.length); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsArray[0]); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsArray[1]); @@ -162,7 +161,7 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); TsKvEntry dataPoint2 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(2), new LongDataEntry("temperature", 43L)); @@ -176,8 +175,8 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - TsValue[] tsArray = listData.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + TsValue[] tsArray = listData.getFirst().getTimeseries().get("temperature"); Assert.assertEquals(3, tsArray.length); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsArray[0]); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsArray[1]); @@ -186,7 +185,7 @@ public class WebsocketApiTest extends AbstractControllerTest { now = System.currentTimeMillis(); TsKvEntry dataPoint4 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 45L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint4)); + sendTelemetry(device, List.of(dataPoint4)); String msg = getWsClient().waitForUpdate(); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); @@ -194,9 +193,9 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getTimeseries()); - TsValue[] tsValues = eData.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getTimeseries()); + TsValue[] tsValues = eData.getFirst().getTimeseries().get("temperature"); Assert.assertNotNull(tsValues); Assert.assertEquals(new TsValue(dataPoint4.getTs(), dataPoint4.getValueAsString()), tsValues[0]); } @@ -521,7 +520,7 @@ public class WebsocketApiTest extends AbstractControllerTest { String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); - // check device + // check a device AlarmStatusCmd deviceCmd = new AlarmStatusCmd(2, device.getId(), null, List.of(AlarmSeverity.CRITICAL)); getWsClient().send(deviceCmd); @@ -589,13 +588,13 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendTelemetry(device, tsData); update = getWsClient().subscribeLatestUpdate(keys); @@ -605,36 +604,36 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); TsKvEntry dataPoint2 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 52L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); Assert.assertEquals(1, update.getCmdId()); List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsValue); //Sending update from the past, while latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint1)); + sendTelemetry(device, List.of(dataPoint1)); String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } @@ -677,14 +676,14 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); getWsClient().registerWaitForUpdate(); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendTelemetry(device, tsData); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); @@ -694,34 +693,34 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); TsKvEntry dataPoint2 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 52L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); Assert.assertEquals(1, update.getCmdId()); List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint1)); + sendTelemetry(device, List.of(dataPoint1)); String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } @@ -736,21 +735,21 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); getWsClient().registerWaitForUpdate(); - // Pushing update with wrong scope and make sure it will not arrive. + // Pushing update with the wrong scope and make sure it will not arrive. AttributeKvEntry invalidDataPoint = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 55L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(invalidDataPoint)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(invalidDataPoint)); Assert.assertNull(getWsClient().waitForUpdate(3000)); AttributeKvEntry dataPoint1 = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData); String msg = getWsClient().waitForUpdate(); @@ -761,16 +760,16 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint1.getLastUpdateTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); AttributeKvEntry dataPoint2 = new BaseAttributeKvEntry(now, new LongDataEntry("serverAttributeKey", 52L)); getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(); Assert.assertNotNull(msg); @@ -779,20 +778,20 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint2.getLastUpdateTs(), dataPoint2.getValueAsString()), tsValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } @@ -812,23 +811,23 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getValue()); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint1 = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData); @@ -839,78 +838,78 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - TsValue attrValue = eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + TsValue attrValue = eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint1.getLastUpdateTs(), dataPoint1.getValueAsString()), attrValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending update from the past, while latest value has new timestamp; getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint2 = new BaseAttributeKvEntry(now, new LongDataEntry("sharedAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey"); Assert.assertEquals(new TsValue(dataPoint2.getLastUpdateTs(), dataPoint2.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint3 = new BaseAttributeKvEntry(now, new LongDataEntry("clientAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint3)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint3)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey"); Assert.assertEquals(new TsValue(dataPoint3.getLastUpdateTs(), dataPoint3.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint4 = new BaseAttributeKvEntry(now, new LongDataEntry("anyAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint4)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint4)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); Assert.assertEquals(new TsValue(dataPoint4.getLastUpdateTs(), dataPoint4.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint5 = new BaseAttributeKvEntry(now, new LongDataEntry("anyAttributeKey", 43L)); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint5)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint5)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); Assert.assertEquals(new TsValue(dataPoint5.getLastUpdateTs(), dataPoint5.getValueAsString()), attrValue); } @@ -971,7 +970,7 @@ public class WebsocketApiTest extends AbstractControllerTest { .tenantId(device.getTenantId()) .entityId(device.getId()) .entries(tsData) - .callback(new FutureCallback() { + .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void result) { log.debug("sendTelemetry callback onSuccess"); diff --git a/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java b/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java index 053cb6808f..accbbd1d29 100644 --- a/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java @@ -25,16 +25,25 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.adapter.NativeWebSocketSession; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; +import org.thingsboard.server.service.security.auth.pat.ApiKeyAuthenticationProvider; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; +import org.thingsboard.server.service.ws.WebSocketSessionType; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Collection; import java.util.Deque; import java.util.List; import java.util.Random; +import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -184,4 +193,101 @@ class TbWebSocketHandlerTest { assertThat(msgs).map(Integer::parseInt).doesNotHaveDuplicates().hasSize(100); } + private AuthTestFixture createAuthTestFixture() throws IOException { + TbWebSocketHandler handler = spy(new TbWebSocketHandler()); + willDoNothing().given(handler).close(any(), any()); + + ApiKeyAuthenticationProvider apiKeyProvider = mock(ApiKeyAuthenticationProvider.class); + JwtAuthenticationProvider jwtProvider = mock(JwtAuthenticationProvider.class); + WebSocketService wsService = mock(WebSocketService.class); + + ReflectionTestUtils.setField(handler, "apiKeyAuthenticationProvider", apiKeyProvider); + ReflectionTestUtils.setField(handler, "authenticationProvider", jwtProvider); + ReflectionTestUtils.setField(handler, "webSocketService", wsService); + ReflectionTestUtils.setField(handler, "tenantProfileCache", mock(TbTenantProfileCache.class)); + ReflectionTestUtils.setField(handler, "authTimeoutMs", 10000); + ReflectionTestUtils.invokeMethod(handler, "init"); + + WebSocketSessionRef ref = WebSocketSessionRef.builder() + .sessionId(UUID.randomUUID().toString()) + .sessionType(WebSocketSessionType.GENERAL) + .build(); + + NativeWebSocketSession wsSession = mock(NativeWebSocketSession.class); + Session nativeSess = mock(Session.class); + willReturn(nativeSess).given(wsSession).getNativeSession(Session.class); + RemoteEndpoint.Async async = mock(RemoteEndpoint.Async.class); + willReturn(async).given(nativeSess).getAsyncRemote(); + willReturn("test-session-id").given(wsSession).getId(); + + TbWebSocketHandler.SessionMetaData sessionMd = handler.new SessionMetaData(wsSession, ref); + + return new AuthTestFixture(handler, apiKeyProvider, jwtProvider, ref, sessionMd); + } + + @Test + void processMsg_authenticatesWithApiKey() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.apiKeyProvider).authenticate("my-api-key"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"apiKey\":\"my-api-key\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.apiKeyProvider).authenticate("my-api-key"); + verify(f.jwtProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void processMsg_authenticatesWithJwtToken() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.jwtProvider).authenticate("my-jwt-token"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"token\":\"my-jwt-token\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.jwtProvider).authenticate("my-jwt-token"); + verify(f.apiKeyProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void processMsg_apiKeyTakesPrecedenceOverToken() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.apiKeyProvider).authenticate("my-api-key"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"apiKey\":\"my-api-key\",\"token\":\"my-jwt-token\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.apiKeyProvider).authenticate("my-api-key"); + verify(f.jwtProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void extractQueryParam_parsesCorrectly() throws Exception { + TbWebSocketHandler handler = new TbWebSocketHandler(); + Method method = TbWebSocketHandler.class.getDeclaredMethod("extractQueryParam", String.class, String.class); + method.setAccessible(true); + + assertThat(method.invoke(handler, "token=jwt123", "token")).isEqualTo("jwt123"); + assertThat(method.invoke(handler, "token=jwt123&other=abc123", "token")).isEqualTo("jwt123"); + assertThat(method.invoke(handler, "other=value", "token")).isNull(); + assertThat(method.invoke(handler, "tokenExtra=value", "token")).isNull(); + } + + private record AuthTestFixture( + TbWebSocketHandler handler, + ApiKeyAuthenticationProvider apiKeyProvider, + JwtAuthenticationProvider jwtProvider, + WebSocketSessionRef ref, + TbWebSocketHandler.SessionMetaData sessionMd + ) {} + } From 163cad4baf22d4d2f22cf00d43db827a418902b6 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 6 Mar 2026 16:47:53 +0200 Subject: [PATCH 14/71] Improve tests --- .../controller/ApiKeyWebSocketApiTest.java | 55 +++++++++++++++++++ .../controller/TbTestWebSocketClient.java | 12 ++++ 2 files changed, 67 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java index b4e96beee2..375d37ae95 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java @@ -17,13 +17,23 @@ package org.thingsboard.server.controller; import lombok.extern.slf4j.Slf4j; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKeyInfo; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; +import java.net.URI; import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -59,6 +69,51 @@ public class ApiKeyWebSocketApiTest extends WebSocketApiTest { TbTestWebSocketClient client = buildAndConnectWebSocketClientWithApiKeyHeader(apiKey.getValue()); try { assertThat(client.isOpen()).isTrue(); + + DeviceTypeFilter dtf = new DeviceTypeFilter(List.of("default"), "Device"); + EntityCountQuery ecq = new EntityCountQuery(dtf, Collections.emptyList()); + EntityCountCmd cmd = new EntityCountCmd(1, ecq); + client.send(cmd); + + EntityCountUpdate update = client.parseCountReply(client.waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertTrue(update.getCount() >= 0); + } finally { + client.close(); + } + } + + @Test + public void testInvalidApiKeyHeader_connectionRejected() throws Exception { + TbTestWebSocketClient client = new TbTestWebSocketClient( + new URI(WS_URL + wsPort + "/api/ws"), Map.of("X-API-Key", "invalid-key")); + try { + boolean connected = client.connectBlocking(TIMEOUT, TimeUnit.SECONDS); + assertThat(connected).isFalse(); + } finally { + client.close(); + } + } + + @Test + public void testEmptyApiKeyHeader_connectionRejected() throws Exception { + TbTestWebSocketClient client = new TbTestWebSocketClient( + new URI(WS_URL + wsPort + "/api/ws"), Map.of("X-API-Key", "")); + try { + boolean connected = client.connectBlocking(TIMEOUT, TimeUnit.SECONDS); + assertThat(connected).isFalse(); + } finally { + client.close(); + } + } + + @Test + public void testInvalidApiKeyAuthCmd_connectionClosed() throws Exception { + TbTestWebSocketClient client = new TbTestWebSocketClient(new URI(WS_URL + wsPort + "/api/ws")); + assertThat(client.connectBlocking(TIMEOUT, TimeUnit.SECONDS)).isTrue(); + try { + client.authenticateWithApiKey("invalid-key"); + assertThat(client.waitForClose()).isTrue(); } finally { client.close(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java index 35536eaecc..d663b2c9ac 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java @@ -53,6 +53,8 @@ public class TbTestWebSocketClient extends WebSocketClient { private static final long TIMEOUT = TimeUnit.SECONDS.toMillis(30); + private final CountDownLatch closeLatch = new CountDownLatch(1); + @Getter private volatile String lastMsg; private volatile CountDownLatch reply; @@ -98,6 +100,16 @@ public class TbTestWebSocketClient extends WebSocketClient { @Override public void onClose(int i, String s, boolean b) { log.info("CLOSED."); + closeLatch.countDown(); + } + + public boolean waitForClose() { + try { + return closeLatch.await(TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + log.warn("Failed to await close", e); + return false; + } } @Override From dcacddcdb644ad52ce291f0da3c0f27496e8f454 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 6 Mar 2026 17:19:01 +0200 Subject: [PATCH 15/71] added env section description --- transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 2bc1e6667d..b4b034a025 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -28,7 +28,8 @@ server: # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8083}" -# Zookeeper connection parameters. Used for service discovery. +# Zookeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" From 8d70ad36bb573c927cb76fc3abf85b1c1b70b8a2 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 6 Mar 2026 17:31:21 +0200 Subject: [PATCH 16/71] added env section description --- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 3 ++- transport/http/src/main/resources/tb-http-transport.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index b52ad1ad89..913ee0801d 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -28,7 +28,8 @@ server: # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8086}" -# Zookeeper connection parameters. Used for service discovery. +# Zookeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:true}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0ce5173ed4..0da6b1d8f4 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -60,7 +60,8 @@ spring.main.allow-circular-references: "true" # Spring Boot configuration proper spring.servlet.multipart.max-file-size: "${SPRING_SERVLET_MULTIPART_MAX_FILE_SIZE:50MB}" # Total file size cannot exceed 50MB when configuring file uploads spring.servlet.multipart.max-request-size: "${SPRING_SERVLET_MULTIPART_MAX_REQUEST_SIZE:50MB}" # Total request size for a multipart/form-data cannot exceed 50MB -# Zookeeper connection parameters. Used for service discovery. +# Zookeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: # Enable/disable zookeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" From 96b742189d25d832e47f95fdd056550629fc6db9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 14:11:33 +0100 Subject: [PATCH 17/71] Fix flaky Sparkplug connection test: handle 404 during device await doGet(url, Class) asserts HTTP 200 internally, so when the Sparkplug device hasn't been created yet the method throws AssertionError instead of returning null. Awaitility propagates Error immediately rather than continuing to poll, causing the test to fail after ~3 s instead of retrying for up to 200 s. Add .ignoreExceptions() to both await() calls in connectClientWithCorrectAccessTokenWithNDEATHCreatedDevices and connectClientWithCorrectAccessTokenWithNDEATHWithAliasCreatedDevices so that a transient 404 is treated as "condition not yet met" and polling continues as intended. Co-Authored-By: Claude Sonnet 4.6 --- .../mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java index 4e4d0debf9..a9bdbb55db 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java @@ -191,6 +191,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; @@ -236,6 +237,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; From 0e3381bca36a644fa97342bd752ebcfb816585e8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 15:26:12 +0100 Subject: [PATCH 18/71] Fix similar flaky await patterns in MQTT transport tests doGet/doGetAsyncTyped assert HTTP 200 internally, so any non-200 response throws AssertionError which Awaitility re-throws immediately instead of continuing to poll. Add .ignoreExceptions() to three additional await() polling loops that call HTTP helpers: - AbstractMqttV5ClientSparkplugAttributesTest: two doGetAsyncTyped calls polling for attribute keys after NBIRTH/DBIRTH - AbstractMqttAttributesIntegrationTest: doGetAsyncTyped polling for attribute values after client publish Co-Authored-By: Claude Sonnet 4.6 --- .../attributes/AbstractMqttAttributesIntegrationTest.java | 1 + .../attributes/AbstractMqttV5ClientSparkplugAttributesTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index a7307b2308..5f43aa7e00 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -420,6 +420,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt Awaitility.await() .atMost(10, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() { }); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java index 756c8e603c..d42adfbcf0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java @@ -468,6 +468,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.NBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); @@ -483,6 +484,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.DBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); From 148dd17ce1e3c1a50bd5ee4d8e4aeb53f4fcccd8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 6 Mar 2026 09:07:06 +0100 Subject: [PATCH 19/71] Fix flaky TenantControllerTest by draining housekeeper before teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests like testFindTenantsByTitle create 261 tenants and delete them via deleteEntitiesAsync, which only waits for HTTP responses — not housekeeper completion. Each tenant deletion submits ~30 TenantEntitiesDeletionHousekeeper tasks (~7800 tasks total), which cascade further. The teardown's deleteTenant then waits for lag==0 with a 90s timeout, which is insufficient for this backlog and causes ConditionTimeoutException. Fix: add awaitHousekeeperDrained() (5-min timeout) called at the start of teardownWebTest so any pending housekeeper work from the test body drains before per-tenant teardown deletions begin. Co-Authored-By: Claude Sonnet 4.6 --- .../thingsboard/server/controller/AbstractWebTest.java | 9 +++++++++ .../src/test/resources/application-test.properties | 1 + 2 files changed, 10 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 3b7286cf01..5f4b7e952f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -405,6 +405,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { public void teardownWebTest() throws Exception { log.debug("Executing web test teardown"); + // Drain any pending housekeeper work left by the test body (e.g., bulk tenant deletes) + // before proceeding with teardown deletions, to avoid 90s per-tenant wait timing out. + awaitHousekeeperDrained(); + loginSysAdmin(); deleteTenant(tenantId); deleteDifferentTenant(); @@ -436,6 +440,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { .until(() -> storage.getLag("tb_housekeeper") == 0); } + protected void awaitHousekeeperDrained() { + Awaitility.await("housekeeper drained").atMost(5, TimeUnit.MINUTES).during(300, TimeUnit.MILLISECONDS) + .until(() -> storage.getLag("tb_housekeeper") == 0); + } + private List getAllTenants() throws Exception { List loadedTenants = new ArrayList<>(); PageLink pageLink = new PageLink(10); diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e79289340c..7f0ab964d6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -44,6 +44,7 @@ queue.transport_api.response_poll_interval=5 queue.transport.poll_interval=5 queue.core.poll-interval=5 queue.core.partitions=2 +queue.core.housekeeper.task-reprocessing-delay-ms=0 queue.rule-engine.poll-interval=5 queue.rule-engine.stats.enabled=true From f73abcdebd1bcfd2b910f97169cd4a6d3bf3b60c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 9 Mar 2026 09:30:31 +0100 Subject: [PATCH 20/71] bump frontend-maven-plugin version to 2.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f00718c322..cd58b11f54 100755 --- a/pom.xml +++ b/pom.xml @@ -655,7 +655,7 @@ com.github.eirslett frontend-maven-plugin - 1.12.0 + 2.0.0 org.apache.maven.plugins From e82861b3e9ae7ce6398e0544bd742dd23627e7af Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 9 Mar 2026 10:50:30 +0200 Subject: [PATCH 21/71] refactoring --- .../subscription/DefaultTbLocalSubscriptionService.java | 4 ++-- .../java/org/thingsboard/server/common/data/kv/TsKvEntry.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index df28da2dad..2b1a810924 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -348,7 +348,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (sub.isLatestValues()) { for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); - if (stateTs == null || kv.getTs() >= stateTs || kv.isDeletedEntryMarker()) { + if (stateTs == null || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } @@ -362,7 +362,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); if (stateTs != null) { - if (!sub.isLatestValues() || kv.getTs() >= stateTs || kv.isDeletedEntryMarker()) { + if (!sub.isLatestValues() || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index eca0609704..595e1aa26b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -38,7 +38,7 @@ public interface TsKvEntry extends KvEntry, HasVersion { } @JsonIgnore - default boolean isDeletedEntryMarker() { + default boolean isDeletedEntry() { return getTs() == 0 && (getValue() == null || getValueAsString().isEmpty()); } From 0587adb789a9f0dfbb47b176fe240f82f781d9b6 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 9 Mar 2026 10:53:07 +0200 Subject: [PATCH 22/71] Refactoring after review --- .../ttl/NotificationsCleanUpService.java | 28 +++++++++---------- .../server/dao/sql/rpc/RpcRepository.java | 2 ++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java index 5e327a72db..83855bc8f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java @@ -62,17 +62,15 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { public void cleanUp() { long expTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttlInSec); long partitionDurationMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); - if (!isSystemTenantPartitionMine()) { + if (isSystemTenantPartitionMine()) { + partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); + } else { partitioningRepository.cleanupPartitionsCache(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - return; } - long lastRemovedNotificationTs = partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - if (lastRemovedNotificationTs > 0) { - long gap = TimeUnit.MINUTES.toMillis(10); - long requestExpTime = lastRemovedNotificationTs - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; - cleanUpNotificationRequests(requestExpTime); - } + long gap = TimeUnit.MINUTES.toMillis(10); + long requestExpTime = expTime - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; + cleanUpNotificationRequests(requestExpTime); } private void cleanUpNotificationRequests(long expirationTime) { @@ -80,13 +78,15 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { int totalRemoved = 0; int tenantsProcessed = 0; - // Clean up SYSADMIN's notification requests: - try { - totalRemoved += cleanUpByTenant(TenantId.SYS_TENANT_ID, expirationTime); - } catch (Exception e) { - log.warn("Failed to clean up notification requests for sysadmin {}", TenantId.SYS_TENANT_ID, e); + // Clean up SYSADMIN's notification requests on the system node only + if (isSystemTenantPartitionMine()) { + try { + totalRemoved += cleanUpByTenant(TenantId.SYS_TENANT_ID, expirationTime); + } catch (Exception e) { + log.warn("Failed to clean up notification requests for sysadmin {}", TenantId.SYS_TENANT_ID, e); + } } - // Clean up notification requests for tenants + // Each node cleans up notification requests for its own tenants PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); for (TenantId tenantId : tenants) { try { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java index 7be106219b..3f76170c84 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.model.sql.RpcEntity; @@ -34,6 +35,7 @@ public interface RpcRepository extends JpaRepository { Page findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying @Query(value = "DELETE FROM rpc WHERE id IN " + "(SELECT id FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime LIMIT :batchSize)", From 1409ee22d41bec1274e893798c79294822eccbe2 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 9 Mar 2026 11:04:57 +0200 Subject: [PATCH 23/71] Fixed ApiKeyEdgeTest --- .../org/thingsboard/server/edge/imitator/EdgeImitator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java index 8cf2b309f6..8208dc4fc9 100644 --- a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java +++ b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java @@ -30,6 +30,7 @@ import org.thingsboard.edge.rpc.EdgeRpcClient; import org.thingsboard.server.controller.AbstractWebTest; import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -375,6 +376,11 @@ public class EdgeImitator { result.add(saveDownlinkMsg(aiModelUpdateMsg)); } } + if (downlinkMsg.getApiKeyUpdateMsgCount() > 0) { + for (ApiKeyUpdateMsg apiKeyUpdateMsg : downlinkMsg.getApiKeyUpdateMsgList()) { + result.add(saveDownlinkMsg(apiKeyUpdateMsg)); + } + } if (downlinkMsg.hasEdgeConfiguration()) { result.add(saveDownlinkMsg(downlinkMsg.getEdgeConfiguration())); } From dd9fdb4181166084a21faa76f91e9708bb98f308 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 9 Mar 2026 11:07:26 +0200 Subject: [PATCH 24/71] refactoring --- .../DefaultTbEntityDataSubscriptionService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 71df830f8d..fc8c5c3be2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -245,7 +245,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc public void onFailure(Throwable t) { log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId(), t); if (t instanceof ResultSetSizeLimitExceededException) { - finalCtx.sendWsMsg(new EntityDataUpdate(finalCtx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + sendError(finalCtx, t); } } }, wsCallBackExecutor); @@ -270,7 +270,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc public void onFailure(Throwable t) { log.warn("[{}][{}] Failed to process timeseries command", ctx.getSessionId(), ctx.getCmdId(), t); if (t instanceof ResultSetSizeLimitExceededException) { - ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + sendError(ctx, t); } } }, wsCallBackExecutor); @@ -283,6 +283,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } } + private void sendError(TbEntityDataSubCtx ctx, Throwable t) { + ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + } + private void checkAndSendInitialData(@Nullable TbEntityDataSubCtx theCtx) { if (!theCtx.isInitialDataSent()) { EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); From 7cce918f90158a10fadc6d070b6f5e07576186e1 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 9 Mar 2026 11:13:38 +0200 Subject: [PATCH 25/71] Change log level to warn --- .../org/thingsboard/server/dao/ota/BaseOtaPackageService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index c43274cc26..54577ff9eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -234,10 +234,10 @@ public class BaseOtaPackageService extends AbstractCachedEntityService Date: Mon, 9 Mar 2026 13:24:46 +0200 Subject: [PATCH 26/71] env descriptions improvements --- application/src/main/resources/thingsboard.yml | 8 ++++---- transport/http/src/main/resources/tb-http-transport.yml | 4 ++-- transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 4 ++-- transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 4 ++-- transport/snmp/src/main/resources/tb-snmp-transport.yml | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1558d1cf74..d7418ede40 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -136,7 +136,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cluster parameters -# Controls inter-node communication statistics within a ThingsBoard cluster. +# Controls cluster statistics — tracks the number of messages exchanged between cluster nodes. cluster: stats: # Enable/Disable the cluster statistics. Calculates the number of messages sent between cluster nodes based on each type @@ -772,9 +772,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # If set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -2107,7 +2107,7 @@ mobileApp: # Link to App Store for Thingsboard Live mobile application appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}" -# MQTT client settings +# MQTT client parameters # Configures MQTT client retransmission behavior including max attempts, initial delay, and jitter factor. mqtt: # MQTT client configuration parameters diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 0da6b1d8f4..7bd55802e3 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -128,9 +128,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index b4b034a025..78185ada98 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -95,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 9298b06d74..f9798e331c 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -96,9 +96,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index a3491170da..74ff4acc1a 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -95,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" From 552bf59680ee23c208bf0301bf5c7a2daf0c1c9b Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 9 Mar 2026 15:39:26 +0200 Subject: [PATCH 27/71] Refactoring ApiKeyService - merge save methods --- .../processor/apikey/BaseApiKeyProcessor.java | 2 +- .../server/dao/pat/ApiKeyService.java | 2 +- .../server/dao/pat/ApiKeyServiceImpl.java | 34 ++++++------------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java index c90ee572bb..7d110c3670 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java @@ -44,7 +44,7 @@ public abstract class BaseApiKeyProcessor extends BaseEdgeProcessor { } apiKey.setId(apiKeyId); - edgeCtx.getApiKeyService().saveApiKey(tenantId, apiKey); + edgeCtx.getApiKeyService().saveApiKey(tenantId, apiKey, apiKey.getValue(), false); } catch (Exception e) { log.error("[{}] Failed to process apiKey update msg [{}]", tenantId, apiKeyUpdateMsg, e); throw e; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java index 19c17127e3..cdb8c23c41 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java @@ -30,7 +30,7 @@ public interface ApiKeyService extends EntityDaoService { ApiKey saveApiKey(TenantId tenantId, ApiKeyInfo apiKey); - ApiKey saveApiKey(TenantId tenantId, ApiKey apiKey); + ApiKey saveApiKey(TenantId tenantId, ApiKeyInfo apiKeyInfo, String value, boolean doValidate); void deleteApiKey(TenantId tenantId, ApiKey apiKey, boolean force); diff --git a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java index c50e13b633..36d8736f78 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java @@ -76,13 +76,20 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService Date: Sat, 30 Aug 2025 08:56:43 +0200 Subject: [PATCH 28/71] MQTTS metrics --- .../transport/mqtt/MqttTransportContext.java | 22 ++++++++++++++----- .../transport/mqtt/MqttTransportHandler.java | 8 +++++-- .../common/transport/TransportService.java | 2 +- .../service/DefaultTransportService.java | 18 ++++++++++++--- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 3d05e999e0..8a60168154 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -88,20 +88,30 @@ public class MqttTransportContext extends TransportContext { @Value("${transport.mqtt.proxy_enabled:false}") private boolean proxyEnabled; - private final AtomicInteger connectionsCounter = new AtomicInteger(); + private final AtomicInteger connectionsActiveCounterMQTT = new AtomicInteger(); + private final AtomicInteger connectionsActiveCounterMQTTS = new AtomicInteger(); @PostConstruct public void init() { super.init(); - transportService.createGaugeStats("openConnections", connectionsCounter); + transportService.createGaugeStats("connections_active", connectionsActiveCounterMQTT, "protocol", "MQTT"); + transportService.createGaugeStats("connections_active", connectionsActiveCounterMQTTS, "protocol", "MQTTS"); } - public void channelRegistered() { - connectionsCounter.incrementAndGet(); + public void channelRegistered(boolean isSSL) { + if (isSSL) { + connectionsActiveCounterMQTTS.incrementAndGet(); + } else { + connectionsActiveCounterMQTT.incrementAndGet(); + } } - public void channelUnregistered() { - connectionsCounter.decrementAndGet(); + public void channelUnregistered(boolean isSSL) { + if (isSSL) { + connectionsActiveCounterMQTTS.decrementAndGet(); + } else { + connectionsActiveCounterMQTT.decrementAndGet(); + } } public boolean checkAddress(InetSocketAddress address) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index f753579292..bf623d67d8 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -180,16 +180,20 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.rpcAwaitingAck = new ConcurrentHashMap<>(); } + boolean isSSL() { + return sslHandler != null; + } + @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { super.channelRegistered(ctx); - context.channelRegistered(); + context.channelRegistered(isSSL()); } @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { super.channelUnregistered(ctx); - context.channelUnregistered(); + context.channelUnregistered(isSSL()); } @Override diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 5c7d552855..612f331757 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -161,5 +161,5 @@ public interface TransportService { boolean hasSession(SessionInfoProto sessionInfo); - void createGaugeStats(String openConnections, AtomicInteger connectionsCounter); + void createGaugeStats(String openConnections, AtomicInteger connectionsCounter, String... tags); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 260957c990..47e341547e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -1257,9 +1257,21 @@ public class DefaultTransportService extends TransportActivityManager implements } @Override - public void createGaugeStats(String statsName, AtomicInteger number) { - statsFactory.createGauge(StatsType.TRANSPORT + "." + statsName, number); - statsMap.put(statsName, number); + public void createGaugeStats(String statsName, AtomicInteger number, String... tags) { + String key = "thingsboard" + "." + StatsType.TRANSPORT.getName() + "." + statsName; + statsFactory.createGauge(key, number, tags); + statsMap.put(statsName + TagsKey(tags), number); + } + + String TagsKey(String... tags) { + if (tags == null || tags.length < 2) return ""; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < tags.length; i += 2) { + if (i > 0) sb.append(','); + sb.append(tags[i]).append('=').append(i + 1 < tags.length ? tags[i + 1] : ""); + } + sb.append(']'); + return sb.toString(); } @Scheduled(fixedDelayString = "${transport.stats.print-interval-ms:60000}") From 2a926cfbdfc5a7c09363a6dec078a211af8cf8d6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 30 Aug 2025 09:39:04 +0200 Subject: [PATCH 29/71] MQTT transport: client remote address logged on exceptionCaught --- .../transport/mqtt/MqttTransportHandler.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index bf623d67d8..aadeff0dee 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -1152,21 +1152,32 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + String clientAddr = null; + try { + InetSocketAddress remote = getAddress(ctx); + clientAddr = remote != null ? (remote.getAddress() != null ? remote.getAddress().getHostAddress() : remote.getHostString()) + ":" + remote.getPort() : "IPunknown"; + } catch (Exception ignored) { + } + if (cause instanceof IOException) { if (log.isDebugEnabled()) { - log.debug("[{}][{}][{}] IOException: {}", sessionId, + log.debug("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), + clientAddr, + cause.getClass().getSimpleName(), cause.getMessage(), cause); } else if (log.isInfoEnabled()) { - log.info("[{}][{}][{}] IOException: {}", sessionId, + log.info("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), + clientAddr, + cause.getClass().getSimpleName(), cause.getMessage()); } } else { - log.error("[{}] Unexpected Exception", sessionId, cause); + log.error("[{}][{}] Unexpected Exception", sessionId, clientAddr, cause); } closeCtx(ctx, MqttReasonCodes.Disconnect.SERVER_SHUTTING_DOWN); From a643a340d8136e4da657488e34a5e840750b1791 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 13:41:05 +0100 Subject: [PATCH 30/71] Address code review: fix naming, visibility, and lazy address resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename TagsKey → tagsKey and make it private (Java naming convention) - Make isSSL() private in MqttTransportHandler (internal use only) - Fix double space in if (isSSL) in MqttTransportContext - Extract getClientAddr() helper and move clientAddr computation inside logging guards so address resolution is skipped when logging is disabled Co-Authored-By: Claude Sonnet 4.6 --- .../transport/mqtt/MqttTransportContext.java | 4 ++-- .../transport/mqtt/MqttTransportHandler.java | 24 ++++++++++++------- .../service/DefaultTransportService.java | 4 ++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 8a60168154..10245c5a24 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -99,7 +99,7 @@ public class MqttTransportContext extends TransportContext { } public void channelRegistered(boolean isSSL) { - if (isSSL) { + if (isSSL) { connectionsActiveCounterMQTTS.incrementAndGet(); } else { connectionsActiveCounterMQTT.incrementAndGet(); @@ -107,7 +107,7 @@ public class MqttTransportContext extends TransportContext { } public void channelUnregistered(boolean isSSL) { - if (isSSL) { + if (isSSL) { connectionsActiveCounterMQTTS.decrementAndGet(); } else { connectionsActiveCounterMQTT.decrementAndGet(); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index aadeff0dee..d427c9b78e 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -180,7 +180,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.rpcAwaitingAck = new ConcurrentHashMap<>(); } - boolean isSSL() { + private boolean isSSL() { return sslHandler != null; } @@ -258,6 +258,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } + private String getClientAddr(ChannelHandlerContext ctx) { + try { + InetSocketAddress remote = getAddress(ctx); + if (remote == null) return "unknown"; + String host = remote.getAddress() != null ? remote.getAddress().getHostAddress() : remote.getHostString(); + return host + ":" + remote.getPort(); + } catch (Exception ignored) { + return "unknown"; + } + } + InetSocketAddress getAddress(ChannelHandlerContext ctx) { var address = ctx.channel().attr(MqttTransportService.ADDRESS).get(); if (address == null) { @@ -1152,15 +1163,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - String clientAddr = null; - try { - InetSocketAddress remote = getAddress(ctx); - clientAddr = remote != null ? (remote.getAddress() != null ? remote.getAddress().getHostAddress() : remote.getHostString()) + ":" + remote.getPort() : "IPunknown"; - } catch (Exception ignored) { - } - if (cause instanceof IOException) { if (log.isDebugEnabled()) { + String clientAddr = getClientAddr(ctx); log.debug("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), @@ -1169,6 +1174,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement cause.getMessage(), cause); } else if (log.isInfoEnabled()) { + String clientAddr = getClientAddr(ctx); log.info("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), @@ -1177,7 +1183,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement cause.getMessage()); } } else { - log.error("[{}][{}] Unexpected Exception", sessionId, clientAddr, cause); + log.error("[{}][{}] Unexpected Exception", sessionId, getClientAddr(ctx), cause); } closeCtx(ctx, MqttReasonCodes.Disconnect.SERVER_SHUTTING_DOWN); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 47e341547e..e8ade498db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -1260,10 +1260,10 @@ public class DefaultTransportService extends TransportActivityManager implements public void createGaugeStats(String statsName, AtomicInteger number, String... tags) { String key = "thingsboard" + "." + StatsType.TRANSPORT.getName() + "." + statsName; statsFactory.createGauge(key, number, tags); - statsMap.put(statsName + TagsKey(tags), number); + statsMap.put(statsName + tagsKey(tags), number); } - String TagsKey(String... tags) { + private String tagsKey(String... tags) { if (tags == null || tags.length < 2) return ""; StringBuilder sb = new StringBuilder("["); for (int i = 0; i < tags.length; i += 2) { From a2393f368c7bcaae76a0d98426987fff6a724180 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Mar 2026 13:42:49 +0100 Subject: [PATCH 31/71] Keep isSSL() and getClientAddr() package-private for test accessibility Methods stubbed via Mockito spy in same-package tests must remain package-private. Revert isSSL() to package-private; getClientAddr() follows the same convention as getAddress(). Co-Authored-By: Claude Sonnet 4.6 --- .../server/transport/mqtt/MqttTransportHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index d427c9b78e..48fbad3032 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -180,7 +180,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.rpcAwaitingAck = new ConcurrentHashMap<>(); } - private boolean isSSL() { + boolean isSSL() { return sslHandler != null; } @@ -258,7 +258,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } - private String getClientAddr(ChannelHandlerContext ctx) { + String getClientAddr(ChannelHandlerContext ctx) { try { InetSocketAddress remote = getAddress(ctx); if (remote == null) return "unknown"; From 2e612899e250b5313d0268c7214dab12475a266b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 10:16:42 +0100 Subject: [PATCH 32/71] Fix NotificationRuleRecipientsConfig serialization with Jackson 2.18.x In Jackson 2.18.x, EXISTING_PROPERTY type info combined with the no-arg @JsonIgnoreProperties causes the triggerType discriminator field to be silently excluded from the serialized JSON. When the server then tries to deserialize the POST body for /api/notification/rule, Jackson cannot find triggerType and throws "missing type id property 'triggerType'", resulting in a 500 for NotificationEdgeTest.testNotificationRule. Fix by: 1. Adding @JsonProperty("triggerType") to force the field into normal bean serialization, overriding any suppression by the type info machinery. 2. Replacing the no-arg @JsonIgnoreProperties with @JsonIgnoreProperties( ignoreUnknown = true) so unknown properties are ignored rather than causing errors (e.g. for forward compatibility). Co-Authored-By: Claude Sonnet 4.6 --- .../notification/rule/NotificationRuleRecipientsConfig.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java index fda73b8059..de4e05a2cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.notification.rule; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -29,7 +30,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; -@JsonIgnoreProperties +@JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "triggerType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = DefaultNotificationRuleRecipientsConfig.class) @JsonSubTypes({ @Type(name = "ALARM", value = EscalatedNotificationRuleRecipientsConfig.class), @@ -38,6 +39,7 @@ import java.util.UUID; public abstract class NotificationRuleRecipientsConfig implements Serializable { @NotNull + @JsonProperty("triggerType") private NotificationRuleTriggerType triggerType; @JsonIgnore From 95016447ccdaf9b120c0885292ed346748166b56 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 11:10:04 +0100 Subject: [PATCH 33/71] Fix EntityViewControllerTest MQTT port collision with other test contexts EntityViewControllerTest was importing MQTT_PORT from AbstractMqttIntegrationTest, a static final field initialized once per JVM. When running in the same Surefire fork alongside other test classes that also use this constant (e.g. MqttGatewayRateLimitsTest, DeviceEdgeTest), each class gets a different Spring context key but all try to bind MqttTransportService to the same port, causing BindException. Fix: define a private static MQTT_PORT/MQTT_URL directly in EntityViewControllerTest so its Spring context gets its own independently allocated port. Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/EntityViewControllerTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 421f3785ca..311ef54fa4 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -41,6 +41,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.TestSocketUtils; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; @@ -87,8 +88,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_PORT; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_URL; @TestPropertySource(properties = { "transport.mqtt.enabled=true", @@ -98,6 +97,9 @@ import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest. @ContextConfiguration(classes = {EntityViewControllerTest.Config.class}) @DaoSqlTest public class EntityViewControllerTest extends AbstractControllerTest { + static final int MQTT_PORT = TestSocketUtils.findAvailableTcpPort(); + static final String MQTT_URL = "tcp://localhost:" + MQTT_PORT; + @DynamicPropertySource static void props(DynamicPropertyRegistry registry) { log.warn("transport.mqtt.bind_port = {}", MQTT_PORT); From 42179bddf8dadf78a242c8943cc81574f9628245 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 11:24:53 +0100 Subject: [PATCH 34/71] Add comment explaining why EntityViewControllerTest owns its MQTT port Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/EntityViewControllerTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 311ef54fa4..b65d4178f0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -97,6 +97,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @ContextConfiguration(classes = {EntityViewControllerTest.Config.class}) @DaoSqlTest public class EntityViewControllerTest extends AbstractControllerTest { + // Must NOT be imported from AbstractMqttIntegrationTest. That field is a static final initialized + // once per JVM. Other test classes (e.g. MqttGatewayRateLimitsTest, DeviceEdgeTest) share the same + // constant but produce a different Spring context cache key, so Spring creates a separate + // ApplicationContext for each of them. Every context starts its own MqttTransportService and tries + // to bind the same port, causing BindException when tests run in the same Surefire JVM fork. + // Declaring the port here gives this context its own independently allocated port. static final int MQTT_PORT = TestSocketUtils.findAvailableTcpPort(); static final String MQTT_URL = "tcp://localhost:" + MQTT_PORT; From da12ee6dc03abbe9dc9ba72fa112e8e82f0306a9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 12:10:26 +0100 Subject: [PATCH 35/71] Fix flaky AuditLogControllerTest: use Awaitility for async audit log assertions Audit logs are saved asynchronously via executor.submit() in AuditLogServiceImpl. Three tests were asserting log counts immediately after HTTP API calls, creating a race condition where the last audit log write may not have completed yet. Replace bare assertions with Awaitility.await().atMost(TIMEOUT, ...).untilAsserted() in testAuditLogs, testAuditLogs_byTenantIdAndEntityId, and testAuditLogs_byTenantIdAndEntityId_Sysadmin (confirmed broken: expected 2, got 1). Also replace the hardcoded 10s timeout in testAuditLogsSysAdmin with the TIMEOUT constant. Co-Authored-By: Claude Sonnet 4.6 --- .../controller/AuditLogControllerTest.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index 728dc1ec0e..56c3ad5619 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -113,17 +113,14 @@ public class AuditLogControllerTest extends AbstractControllerTest { doPost("/api/device", device, Device.class); } - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs?"); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(5, "/api/audit/logs?")).hasSize(11 + 1)); - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(5, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); - loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); - - loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(5, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); } @Test @@ -138,7 +135,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { } int expectedSize = loadedAuditLogsBefore.size() + 3; - Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> Assert.assertEquals("Have X audit log before this test + New tenant profiles in the test", expectedSize, getAuditLogs(100, "/api/audit/logs?").size())); } @@ -170,9 +167,8 @@ public class AuditLogControllerTest extends AbstractControllerTest { savedDevice = doPost("/api/device", savedDevice, Device.class); } - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(5, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); } @Test @@ -188,9 +184,9 @@ public class AuditLogControllerTest extends AbstractControllerTest { tenantProfile.setName(tenantProfile.getName() + "(old)"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/entity/" +tenantProfile.getId().getEntityType()+ "/" + tenantProfile.getId().getId() + "?"); - - Assert.assertEquals("Audit logs count by Tenant Profile entity", 2, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(5, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) + .as("Audit logs count by Tenant Profile entity").hasSize(2)); //cleanup doDelete("/api/tenantProfile/" + tenantProfile.getId().getId().toString()); From feae24fac8098a3392449f30e07182cabe8d8f79 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 12:22:07 +0100 Subject: [PATCH 36/71] Extract PAGE_SIZE constant in AuditLogControllerTest Replace magic number 5 with a named constant that explains intent: small enough to force multiple pages and verify pagination loop correctness. Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/AuditLogControllerTest.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index 56c3ad5619..44e9980f52 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -62,6 +62,9 @@ import static org.mockito.Mockito.verify; @DaoSqlTest public class AuditLogControllerTest extends AbstractControllerTest { + // Small enough to force multiple pages, verifying pagination loop correctness + private static final int PAGE_SIZE = 5; + private Tenant savedTenant; private User tenantAdmin; @@ -114,13 +117,13 @@ public class AuditLogControllerTest extends AbstractControllerTest { } Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(5, "/api/audit/logs?")).hasSize(11 + 1)); + assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs?")).hasSize(11 + 1)); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(5, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(5, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); } @Test @@ -168,7 +171,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { } Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(5, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); } @Test @@ -185,7 +188,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(5, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) + assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) .as("Audit logs count by Tenant Profile entity").hasSize(2)); //cleanup From 611004327dab9b5a39b313ce109de6e50368def1 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 12:22:55 +0100 Subject: [PATCH 37/71] Rename PAGE_SIZE to SMALL_PAGE_SIZE for clarity Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/AuditLogControllerTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index 44e9980f52..35804c1bd4 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -63,7 +63,7 @@ import static org.mockito.Mockito.verify; public class AuditLogControllerTest extends AbstractControllerTest { // Small enough to force multiple pages, verifying pagination loop correctness - private static final int PAGE_SIZE = 5; + private static final int SMALL_PAGE_SIZE = 5; private Tenant savedTenant; private User tenantAdmin; @@ -117,13 +117,13 @@ public class AuditLogControllerTest extends AbstractControllerTest { } Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs?")).hasSize(11 + 1)); + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs?")).hasSize(11 + 1)); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); } @Test @@ -171,7 +171,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { } Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); } @Test @@ -188,7 +188,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(PAGE_SIZE, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) .as("Audit logs count by Tenant Profile entity").hasSize(2)); //cleanup From 26acf60768f816cd2d6bde75b622089b3e7b6ac7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 10 Mar 2026 13:20:40 +0100 Subject: [PATCH 38/71] Fix compilation error: use effectively final variables in lambda expressions Variables reassigned in a loop cannot be captured in lambda expressions. Introduce finalSavedDevice and finalTenantProfile as effectively final copies before use in Awaitility.await() lambdas. Co-Authored-By: Claude Sonnet 4.6 --- .../server/controller/AuditLogControllerTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index 35804c1bd4..95582cf261 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -170,8 +170,9 @@ public class AuditLogControllerTest extends AbstractControllerTest { savedDevice = doPost("/api/device", savedDevice, Device.class); } + Device finalSavedDevice = savedDevice; Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?")).hasSize(11 + 1)); + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + finalSavedDevice.getId().getId() + "?")).hasSize(11 + 1)); } @Test @@ -187,8 +188,9 @@ public class AuditLogControllerTest extends AbstractControllerTest { tenantProfile.setName(tenantProfile.getName() + "(old)"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + TenantProfile finalTenantProfile = tenantProfile; Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> - assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/" + tenantProfile.getId().getEntityType() + "/" + tenantProfile.getId().getId() + "?")) + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/" + finalTenantProfile.getId().getEntityType() + "/" + finalTenantProfile.getId().getId() + "?")) .as("Audit logs count by Tenant Profile entity").hasSize(2)); //cleanup From bd181f115ed8ea536a551bfc3fd984f3527a5548 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 10 Mar 2026 17:26:15 +0200 Subject: [PATCH 39/71] UI: Hidden show on widgets button in Sys Admin users --- .../components/attribute/attribute-table.component.html | 3 ++- .../home/components/attribute/attribute-table.component.ts | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html index 6a797a37aa..6d758b8c7b 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html @@ -93,7 +93,8 @@ (click)="deleteTelemetry($event)"> delete -