From a098bf0bf902ffa06d60da8ce6de870dfd656459 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 6 Oct 2021 18:05:02 +0300 Subject: [PATCH 01/34] Implement concurrent bulk import processing --- .../server/controller/AssetController.java | 11 +-- .../server/controller/DeviceController.java | 11 +-- .../server/controller/EdgeController.java | 8 +- .../device/DeviceBulkImportService.java | 53 +++++++----- .../importing/AbstractBulkImportService.java | 80 +++++++++++++------ .../service/importing/BulkImportResult.java | 14 ++-- .../service/importing/ImportedEntityInfo.java | 1 - .../thingsboard/common/util/DonAsynchron.java | 12 +++ .../dao/device/DeviceCredentialsDao.java | 2 + .../device/DeviceCredentialsServiceImpl.java | 2 +- .../server/dao/device/DeviceProfileDao.java | 2 + .../dao/device/DeviceProfileServiceImpl.java | 2 +- .../device/DeviceCredentialsRepository.java | 4 +- .../sql/device/DeviceProfileRepository.java | 3 +- .../sql/device/JpaDeviceCredentialsDao.java | 9 +++ .../dao/sql/device/JpaDeviceProfileDao.java | 9 +++ 16 files changed, 152 insertions(+), 71 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index d63674b7ee..a4e1af1da5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -116,7 +116,7 @@ public class AssetController extends BaseController { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - onAssetCreatedOrUpdated(savedAsset, asset.getId() != null); + onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser()); return savedAsset; } catch (Exception e) { @@ -126,9 +126,9 @@ public class AssetController extends BaseController { } } - private void onAssetCreatedOrUpdated(Asset asset, boolean updated) { + private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) { try { - logEntityAction(asset.getId(), asset, + logEntityAction(user, asset.getId(), asset, asset.getCustomerId(), updated ? ActionType.UPDATED : ActionType.ADDED, null); } catch (ThingsboardException e) { @@ -550,8 +550,9 @@ public class AssetController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @PostMapping("/asset/bulk_import") public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { - return assetBulkImportService.processBulkImport(request, getCurrentUser(), importedAssetInfo -> { - onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated()); + SecurityUser user = getCurrentUser(); + return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { + onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 8cfa5ee2cf..fbd03b3067 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -141,7 +141,7 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created); + onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser()); return savedDevice; } catch (Exception e) { @@ -152,11 +152,11 @@ public class DeviceController extends BaseController { } - private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated) { + private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) { tbClusterService.onDeviceUpdated(savedDevice, oldDevice); try { - logEntityAction(savedDevice.getId(), savedDevice, + logEntityAction(user, savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), updated ? ActionType.UPDATED : ActionType.ADDED, null); } catch (ThingsboardException e) { @@ -796,8 +796,9 @@ public class DeviceController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @PostMapping("/device/bulk_import") public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { - return deviceBulkImportService.processBulkImport(request, getCurrentUser(), importedDeviceInfo -> { - onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated()); + SecurityUser user = getCurrentUser(); + return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { + onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 21c39cdd58..da0e103ad2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -140,7 +140,7 @@ public class EdgeController extends BaseController { edge.getId(), edge); Edge savedEdge = checkNotNull(edgeService.saveEdge(edge, true)); - onEdgeCreatedOrUpdated(tenantId, savedEdge, edgeTemplateRootRuleChain, !created); + onEdgeCreatedOrUpdated(tenantId, savedEdge, edgeTemplateRootRuleChain, !created, getCurrentUser()); return savedEdge; } catch (Exception e) { @@ -150,7 +150,7 @@ public class EdgeController extends BaseController { } } - private void onEdgeCreatedOrUpdated(TenantId tenantId, Edge edge, RuleChain edgeTemplateRootRuleChain, boolean updated) throws IOException, ThingsboardException { + private void onEdgeCreatedOrUpdated(TenantId tenantId, Edge edge, RuleChain edgeTemplateRootRuleChain, boolean updated, SecurityUser user) throws IOException, ThingsboardException { if (!updated) { ruleChainService.assignRuleChainToEdge(tenantId, edgeTemplateRootRuleChain.getId(), edge.getId()); edgeNotificationService.setEdgeRootRuleChain(tenantId, edge, edgeTemplateRootRuleChain.getId()); @@ -160,7 +160,7 @@ public class EdgeController extends BaseController { tbClusterService.broadcastEntityStateChangeEvent(edge.getTenantId(), edge.getId(), updated ? ComponentLifecycleEvent.UPDATED : ComponentLifecycleEvent.CREATED); - logEntityAction(edge.getId(), edge, null, updated ? ActionType.UPDATED : ActionType.ADDED, null); + logEntityAction(user, edge.getId(), edge, null, updated ? ActionType.UPDATED : ActionType.ADDED, null); } @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -586,7 +586,7 @@ public class EdgeController extends BaseController { return edgeBulkImportService.processBulkImport(request, user, importedAssetInfo -> { try { - onEdgeCreatedOrUpdated(user.getTenantId(), importedAssetInfo.getEntity(), edgeTemplateRootRuleChain, importedAssetInfo.isUpdated()); + onEdgeCreatedOrUpdated(user.getTenantId(), importedAssetInfo.getEntity(), edgeTemplateRootRuleChain, importedAssetInfo.isUpdated(), user); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 82d4dc5571..707674f6ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -63,6 +63,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; @Service @TbCoreComponent @@ -71,6 +73,8 @@ public class DeviceBulkImportService extends AbstractBulkImportService { protected final DeviceCredentialsService deviceCredentialsService; protected final DeviceProfileService deviceProfileService; + private final Lock findOrCreateDeviceProfileLock = new ReentrantLock(); + public DeviceBulkImportService(TelemetrySubscriptionService tsSubscriptionService, TbTenantProfileCache tenantProfileCache, AccessControlService accessControlService, AccessValidator accessValidator, EntityActionService entityActionService, TbClusterService clusterService, @@ -106,9 +110,13 @@ public class DeviceBulkImportService extends AbstractBulkImportService { throw new DeviceCredentialsValidationException("Invalid device credentials: " + e.getMessage()); } + DeviceProfile deviceProfile; if (deviceCredentials.getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) { - setUpLwM2mDeviceProfile(user.getTenantId(), device); + deviceProfile = setUpLwM2mDeviceProfile(user.getTenantId(), device); + } else { + deviceProfile = deviceProfileService.findOrCreateDeviceProfile(user.getTenantId(), device.getType()); } + device.setDeviceProfileId(deviceProfile.getId()); device = deviceService.saveDeviceWithCredentials(device, deviceCredentials); @@ -215,36 +223,43 @@ public class DeviceBulkImportService extends AbstractBulkImportService { credentials.setCredentialsValue(lwm2mCredentials.toString()); } - private void setUpLwM2mDeviceProfile(TenantId tenantId, Device device) { + private DeviceProfile setUpLwM2mDeviceProfile(TenantId tenantId, Device device) { DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); if (deviceProfile != null) { if (deviceProfile.getTransportType() != DeviceTransportType.LWM2M) { deviceProfile.setTransportType(DeviceTransportType.LWM2M); deviceProfile.getProfileData().setTransportConfiguration(new Lwm2mDeviceProfileTransportConfiguration()); deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - device.setDeviceProfileId(deviceProfile.getId()); } } else { - deviceProfile = new DeviceProfile(); - deviceProfile.setTenantId(tenantId); - deviceProfile.setType(DeviceProfileType.DEFAULT); - deviceProfile.setName(device.getType()); - deviceProfile.setTransportType(DeviceTransportType.LWM2M); - deviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED); + findOrCreateDeviceProfileLock.lock(); + try { + deviceProfile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); + if (deviceProfile == null) { + deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setName(device.getType()); + deviceProfile.setTransportType(DeviceTransportType.LWM2M); + deviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED); - DeviceProfileData deviceProfileData = new DeviceProfileData(); - DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); - DeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration(); - DisabledDeviceProfileProvisionConfiguration provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(null); + DeviceProfileData deviceProfileData = new DeviceProfileData(); + DefaultDeviceProfileConfiguration configuration = new DefaultDeviceProfileConfiguration(); + DeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration(); + DisabledDeviceProfileProvisionConfiguration provisionConfiguration = new DisabledDeviceProfileProvisionConfiguration(null); - deviceProfileData.setConfiguration(configuration); - deviceProfileData.setTransportConfiguration(transportConfiguration); - deviceProfileData.setProvisionConfiguration(provisionConfiguration); - deviceProfile.setProfileData(deviceProfileData); + deviceProfileData.setConfiguration(configuration); + deviceProfileData.setTransportConfiguration(transportConfiguration); + deviceProfileData.setProvisionConfiguration(provisionConfiguration); + deviceProfile.setProfileData(deviceProfileData); - deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - device.setDeviceProfileId(deviceProfile.getId()); + deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + } + } finally { + findOrCreateDeviceProfileLock.unlock(); + } } + return deviceProfile; } private void setValues(ObjectNode objectNode, Map data, Collection columns) { diff --git a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java index b1d0b30c9d..a1e3ff16ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java @@ -22,6 +22,9 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.TenantProfile; @@ -47,11 +50,16 @@ import org.thingsboard.server.utils.CsvUtils; import org.thingsboard.server.utils.TypeCastUtil; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -67,39 +75,49 @@ public abstract class AbstractBulkImportService processBulkImport(BulkImportRequest request, SecurityUser user, Consumer> onEntityImported) throws Exception { - BulkImportResult result = new BulkImportResult<>(); + private static ThreadPoolExecutor executor; - AtomicInteger i = new AtomicInteger(0); - if (request.getMapping().getHeader()) { - i.incrementAndGet(); + @PostConstruct + private void initExecutor() { + if (executor == null) { + executor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), + 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(150_000), + ThingsBoardThreadFactory.forName("bulk-import"), new ThreadPoolExecutor.CallerRunsPolicy()); + executor.allowCoreThreadTimeOut(true); } + } - parseData(request).forEach(entityData -> { - i.incrementAndGet(); - try { - ImportedEntityInfo importedEntityInfo = saveEntity(request, entityData.getFields(), user); - onEntityImported.accept(importedEntityInfo); + public final BulkImportResult processBulkImport(BulkImportRequest request, SecurityUser user, Consumer> onEntityImported) throws Exception { + List entitiesData = parseData(request); - E entity = importedEntityInfo.getEntity(); + BulkImportResult result = new BulkImportResult<>(); + CountDownLatch completionLatch = new CountDownLatch(entitiesData.size()); - saveKvs(user, entity, entityData.getKvs()); + entitiesData.forEach(entityData -> DonAsynchron.submit(() -> { + ImportedEntityInfo importedEntityInfo = saveEntity(request, entityData.getFields(), user); + E entity = importedEntityInfo.getEntity(); - if (importedEntityInfo.getRelatedError() != null) { - throw new RuntimeException(importedEntityInfo.getRelatedError()); - } + onEntityImported.accept(importedEntityInfo); + saveKvs(user, entity, entityData.getKvs()); - if (importedEntityInfo.isUpdated()) { - result.setUpdated(result.getUpdated() + 1); - } else { - result.setCreated(result.getCreated() + 1); - } - } catch (Exception e) { - result.setErrors(result.getErrors() + 1); - result.getErrorsList().add(String.format("Line %d: %s", i.get(), e.getMessage())); - } - }); + return importedEntityInfo; + }, + importedEntityInfo -> { + if (importedEntityInfo.isUpdated()) { + result.getUpdated().incrementAndGet(); + } else { + result.getCreated().incrementAndGet(); + } + completionLatch.countDown(); + }, + throwable -> { + result.getErrors().incrementAndGet(); + result.getErrorsList().add(String.format("Line %d: %s", entityData.getLineNumber(), ExceptionUtils.getRootCauseMessage(throwable))); + completionLatch.countDown(); + }, + executor)); + completionLatch.await(); return result; } @@ -186,8 +204,11 @@ public abstract class AbstractBulkImportService parseData(BulkImportRequest request) throws Exception { List> records = CsvUtils.parseCsv(request.getFile(), request.getMapping().getDelimiter()); + AtomicInteger linesCounter = new AtomicInteger(0); + if (request.getMapping().getHeader()) { records.remove(0); + linesCounter.incrementAndGet(); } List columnsMappings = request.getMapping().getColumns(); @@ -205,15 +226,24 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); private final Map kvs = new LinkedHashMap<>(); + private int lineNumber; } @Data diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java index d6fa6ccbf9..7e937d835d 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java @@ -17,14 +17,14 @@ package org.thingsboard.server.service.importing; import lombok.Data; -import java.util.LinkedList; -import java.util.List; +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicInteger; @Data public class BulkImportResult { - private int created = 0; - private int updated = 0; - private int errors = 0; - private List errorsList = new LinkedList<>(); - + private AtomicInteger created = new AtomicInteger(); + private AtomicInteger updated = new AtomicInteger(); + private AtomicInteger errors = new AtomicInteger(); + private Collection errorsList = new ConcurrentLinkedDeque<>(); } diff --git a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java index 958863537a..846444c1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java @@ -22,5 +22,4 @@ public class ImportedEntityInfo { private E entity; private boolean isUpdated; private E oldEntity; - private String relatedError; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java index 5480a3bd69..d615890507 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import java.util.concurrent.Callable; import java.util.concurrent.Executor; import java.util.function.Consumer; @@ -53,4 +54,15 @@ public class DonAsynchron { Futures.addCallback(future, callback, MoreExecutors.directExecutor()); } } + + public static ListenableFuture submit(Callable task, Consumer onSuccess, Consumer onFailure, Executor executor) { + return submit(task, onSuccess, onFailure, executor, null); + } + + public static ListenableFuture submit(Callable task, Consumer onSuccess, Consumer onFailure, Executor executor, Executor callbackExecutor) { + ListenableFuture future = Futures.submit(task, executor); + withCallback(future, onSuccess, onFailure, callbackExecutor); + return future; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsDao.java index 59ed00052e..7e07347ca4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsDao.java @@ -35,6 +35,8 @@ public interface DeviceCredentialsDao extends Dao { */ DeviceCredentials save(TenantId tenantId, DeviceCredentials deviceCredentials); + DeviceCredentials saveAndFlush(TenantId tenantId, DeviceCredentials deviceCredentials); + /** * Find device credentials by device id. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java index 47af5df8fe..14539066ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java @@ -96,7 +96,7 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen log.trace("Executing updateDeviceCredentials [{}]", deviceCredentials); credentialsValidator.validate(deviceCredentials, id -> tenantId); try { - return deviceCredentialsDao.save(tenantId, deviceCredentials); + return deviceCredentialsDao.saveAndFlush(tenantId, deviceCredentials); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index 120bf0bad0..2200416e6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -30,6 +30,8 @@ public interface DeviceProfileDao extends Dao { DeviceProfile save(TenantId tenantId, DeviceProfile deviceProfile); + DeviceProfile saveAndFlush(TenantId tenantId, DeviceProfile deviceProfile); + PageData findDeviceProfiles(TenantId tenantId, PageLink pageLink); PageData findDeviceProfileInfos(TenantId tenantId, PageLink pageLink, String transportType); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 720d50ad69..105352c8a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -167,7 +167,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } DeviceProfile savedDeviceProfile; try { - savedDeviceProfile = deviceProfileDao.save(deviceProfile.getTenantId(), deviceProfile); + savedDeviceProfile = deviceProfileDao.saveAndFlush(deviceProfile.getTenantId(), deviceProfile); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("device_profile_name_unq_key")) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 02ae32bd3e..97c9913239 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.dao.sql.device; -import org.springframework.data.repository.CrudRepository; +import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import java.util.UUID; @@ -23,7 +23,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface DeviceCredentialsRepository extends CrudRepository { +public interface DeviceCredentialsRepository extends JpaRepository { DeviceCredentialsEntity findByDeviceId(UUID deviceId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 1dc2af4ecf..311c73100c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; @@ -26,7 +27,7 @@ import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import java.util.UUID; -public interface DeviceProfileRepository extends PagingAndSortingRepository { +public interface DeviceProfileRepository extends JpaRepository { @Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " + "FROM DeviceProfileEntity d " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 68eb7ab37a..26453a5e52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.DaoUtil; @@ -46,6 +47,14 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao findDeviceProfiles(TenantId tenantId, PageLink pageLink) { return DaoUtil.toPageData( From 2814c377acd45b3f8cb75955ede0fc5fb722261a Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 11 Oct 2021 16:29:46 +0300 Subject: [PATCH 02/34] Refactor rule chains importing --- .../controller/RuleChainController.java | 13 ++-- .../server/dao/rule/RuleChainService.java | 3 +- .../data/rule/RuleChainImportResult.java | 13 ++-- .../server/dao/rule/BaseRuleChainService.java | 59 +++++++++++-------- .../server/dao/rule/RuleChainDao.java | 5 ++ .../server/dao/sql/rule/JpaRuleChainDao.java | 7 +++ .../dao/sql/rule/RuleChainRepository.java | 8 ++- 7 files changed, 68 insertions(+), 40 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index a71fb70e0a..2fcd36f95b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -450,15 +449,17 @@ public class RuleChainController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/ruleChains/import", method = RequestMethod.POST) @ResponseBody - public void importRuleChains(@RequestBody RuleChainData ruleChainData, @RequestParam(required = false, defaultValue = "false") boolean overwrite) throws ThingsboardException { + public List importRuleChains(@RequestBody RuleChainData ruleChainData, @RequestParam(required = false, defaultValue = "false") boolean overwrite) throws ThingsboardException { try { TenantId tenantId = getCurrentUser().getTenantId(); - List importResults = ruleChainService.importTenantRuleChains(tenantId, ruleChainData, RuleChainType.CORE, overwrite); - if (!CollectionUtils.isEmpty(importResults)) { - for (RuleChainImportResult importResult : importResults) { - tbClusterService.broadcastEntityStateChangeEvent(importResult.getTenantId(), importResult.getRuleChainId(), importResult.getLifecycleEvent()); + List importResults = ruleChainService.importTenantRuleChains(tenantId, ruleChainData, overwrite); + for (RuleChainImportResult importResult : importResults) { + if (importResult.getError() == null) { + tbClusterService.broadcastEntityStateChangeEvent(importResult.getTenantId(), importResult.getRuleChainId(), + importResult.isUpdated() ? ComponentLifecycleEvent.UPDATED : ComponentLifecycleEvent.CREATED); } } + return importResults; } catch (Exception e) { throw handleException(e); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index ae994d4122..e089d7bf47 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; @@ -71,7 +70,7 @@ public interface RuleChainService { RuleChainData exportTenantRuleChains(TenantId tenantId, PageLink pageLink) throws ThingsboardException; - List importTenantRuleChains(TenantId tenantId, RuleChainData ruleChainData, RuleChainType type, boolean overwrite); + List importTenantRuleChains(TenantId tenantId, RuleChainData ruleChainData, boolean overwrite); RuleChain assignRuleChainToEdge(TenantId tenantId, RuleChainId ruleChainId, EdgeId edgeId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChainImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChainImportResult.java index 11a2f68a57..70c30de947 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChainImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChainImportResult.java @@ -15,17 +15,22 @@ */ package org.thingsboard.server.common.data.rule; -import lombok.AllArgsConstructor; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @Data -@AllArgsConstructor public class RuleChainImportResult { + @JsonIgnore private TenantId tenantId; private RuleChainId ruleChainId; - private ComponentLifecycleEvent lifecycleEvent; + private String ruleChainName; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private boolean updated; + @JsonInclude(JsonInclude.Include.NON_NULL) + private String error; + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 2ead3f4334..fa1c4ec173 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -38,7 +39,6 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.NodeConnectionInfo; @@ -59,6 +59,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantDao; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -416,41 +417,46 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } @Override - public List importTenantRuleChains(TenantId tenantId, RuleChainData ruleChainData, RuleChainType type, boolean overwrite) { + public List importTenantRuleChains(TenantId tenantId, RuleChainData ruleChainData, boolean overwrite) { List importResults = new ArrayList<>(); + setRandomRuleChainIds(ruleChainData); resetRuleNodeIds(ruleChainData.getMetadata()); resetRuleChainMetadataTenantIds(tenantId, ruleChainData.getMetadata()); - if (overwrite) { - List persistentRuleChains = findAllTenantRuleChains(tenantId, type); - for (RuleChain ruleChain : ruleChainData.getRuleChains()) { - ComponentLifecycleEvent lifecycleEvent; - Optional persistentRuleChainOpt = persistentRuleChains.stream().filter(rc -> rc.getName().equals(ruleChain.getName())).findFirst(); - if (persistentRuleChainOpt.isPresent()) { - setNewRuleChainId(ruleChain, ruleChainData.getMetadata(), ruleChain.getId(), persistentRuleChainOpt.get().getId()); - ruleChain.setRoot(persistentRuleChainOpt.get().isRoot()); - lifecycleEvent = ComponentLifecycleEvent.UPDATED; - } else { - ruleChain.setRoot(false); - lifecycleEvent = ComponentLifecycleEvent.CREATED; + + for (RuleChain ruleChain : ruleChainData.getRuleChains()) { + RuleChainImportResult importResult = new RuleChainImportResult(); + + ruleChain.setTenantId(tenantId); + ruleChain.setRoot(false); + + if (overwrite) { + Collection existingRuleChains = ruleChainDao.findByTenantIdAndTypeAndName(tenantId, + Optional.ofNullable(ruleChain.getType()).orElse(RuleChainType.CORE), ruleChain.getName()); + Optional existingRuleChain = existingRuleChains.stream().findFirst(); + if (existingRuleChain.isPresent()) { + setNewRuleChainId(ruleChain, ruleChainData.getMetadata(), ruleChain.getId(), existingRuleChain.get().getId()); + ruleChain.setRoot(existingRuleChain.get().isRoot()); + importResult.setUpdated(true); } - ruleChain.setTenantId(tenantId); - ruleChainDao.save(tenantId, ruleChain); - importResults.add(new RuleChainImportResult(tenantId, ruleChain.getId(), lifecycleEvent)); } - } else { - if (!CollectionUtils.isEmpty(ruleChainData.getRuleChains())) { - ruleChainData.getRuleChains().forEach(rc -> { - rc.setTenantId(tenantId); - rc.setRoot(false); - RuleChain savedRc = ruleChainDao.save(tenantId, rc); - importResults.add(new RuleChainImportResult(tenantId, savedRc.getId(), ComponentLifecycleEvent.CREATED)); - }); + + try { + ruleChain = saveRuleChain(ruleChain); + } catch (Exception e) { + importResult.setError(ExceptionUtils.getRootCauseMessage(e)); } + + importResult.setTenantId(tenantId); + importResult.setRuleChainId(ruleChain.getId()); + importResult.setRuleChainName(ruleChain.getName()); + importResults.add(importResult); } - if (!CollectionUtils.isEmpty(ruleChainData.getMetadata())) { + + if (CollectionUtils.isNotEmpty(ruleChainData.getMetadata())) { ruleChainData.getMetadata().forEach(md -> saveRuleChainMetaData(tenantId, md)); } + return importResults; } @@ -723,4 +729,5 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC checkRuleNodesAndDelete(tenantId, entity.getId()); } }; + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 98ad0b34a7..03fd25e944 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; @@ -22,6 +23,7 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; +import java.util.Collection; import java.util.UUID; /** @@ -74,4 +76,7 @@ public interface RuleChainDao extends Dao, TenantEntityDao { * @return the list of rule chain objects */ PageData findAutoAssignToEdgeRuleChainsByTenantId(UUID tenantId, PageLink pageLink); + + Collection findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 040e60daad..483ab39f06 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.dao.model.sql.RuleChainEntity; import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; +import java.util.Collection; import java.util.Objects; import java.util.UUID; @@ -97,8 +98,14 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name) { + return DaoUtil.convertDataList(ruleChainRepository.findByTenantIdAndTypeAndName(tenantId.getId(), type, name)); + } + @Override public Long countByTenantId(TenantId tenantId) { return ruleChainRepository.countByTenantId(tenantId.getId()); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 86a147cc57..fda0a0d7f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -23,6 +23,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.model.sql.RuleChainEntity; +import java.util.List; import java.util.UUID; public interface RuleChainRepository extends PagingAndSortingRepository { @@ -55,10 +56,13 @@ public interface RuleChainRepository extends PagingAndSortingRepository findAutoAssignByTenantId(@Param("tenantId") UUID tenantId, - @Param("searchText") String searchText, - Pageable pageable); + @Param("searchText") String searchText, + Pageable pageable); RuleChainEntity findByTenantIdAndTypeAndRootIsTrue(UUID tenantId, RuleChainType ruleChainType); Long countByTenantId(UUID tenantId); + + List findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + } From 5ad33bafc6052f41d4ee223e21308687bd1aada7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 24 Sep 2021 10:45:54 +0300 Subject: [PATCH 03/34] clear alarm node: default script changed to avoid infinite metadata grow --- .../engine/action/TbClearAlarmNodeConfiguration.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java index 3de9ca96c7..b2641cfdd5 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java @@ -25,10 +25,18 @@ public class TbClearAlarmNodeConfiguration extends TbAbstractAlarmNodeConfigurat @Override public TbClearAlarmNodeConfiguration defaultConfiguration() { TbClearAlarmNodeConfiguration configuration = new TbClearAlarmNodeConfiguration(); - configuration.setAlarmDetailsBuildJs("var details = {};\n" + + configuration.setAlarmDetailsBuildJs("" + + "//***DO NOT CHANGE THIS LINES***\n" + + "var details = {};\n" + "if (metadata.prevAlarmDetails) {\n" + " details = JSON.parse(metadata.prevAlarmDetails);\n" + + " //remove prevAlarmDetails from metadata\n" + + " metadata.delete('prevAlarmDetails')\n" + + " //now metadata is the same as it comes to this rule node" + "}\n" + + "//***PLACE YOU CODE BELOW***\n" + + "\n" + + "\n" + "return details;"); configuration.setAlarmType("General Alarm"); return configuration; From 6d468036e75555cfa1804b610a98a5ee4dd6dab2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 12 Oct 2021 14:46:03 +0300 Subject: [PATCH 04/34] clear alarm node: fixed typo --- .../rule/engine/action/TbClearAlarmNodeConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java index b2641cfdd5..6bc3169487 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java @@ -32,9 +32,9 @@ public class TbClearAlarmNodeConfiguration extends TbAbstractAlarmNodeConfigurat " details = JSON.parse(metadata.prevAlarmDetails);\n" + " //remove prevAlarmDetails from metadata\n" + " metadata.delete('prevAlarmDetails')\n" + - " //now metadata is the same as it comes to this rule node" + + " //now metadata is the same as it comes IN this rule node" + "}\n" + - "//***PLACE YOU CODE BELOW***\n" + + "//***PLACE YOUR CODE BELOW***\n" + "\n" + "\n" + "return details;"); From 63d33ba1bfb1f186e2abee9b58762c5566132f2a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 12 Oct 2021 15:40:29 +0300 Subject: [PATCH 05/34] clear alarm node: fixed "delete metadata.prevAlarmDetails" --- .../rule/engine/action/TbClearAlarmNodeConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java index 6bc3169487..24736136f2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNodeConfiguration.java @@ -31,7 +31,7 @@ public class TbClearAlarmNodeConfiguration extends TbAbstractAlarmNodeConfigurat "if (metadata.prevAlarmDetails) {\n" + " details = JSON.parse(metadata.prevAlarmDetails);\n" + " //remove prevAlarmDetails from metadata\n" + - " metadata.delete('prevAlarmDetails')\n" + + " delete metadata.prevAlarmDetails;\n" + " //now metadata is the same as it comes IN this rule node" + "}\n" + "//***PLACE YOUR CODE BELOW***\n" + From 18800c57c8ec14955f32fed13100c76e570e21da Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 27 Aug 2021 17:45:50 +0300 Subject: [PATCH 06/34] tests: added first test to the common/data package (maven plugin added as well) --- common/data/pom.xml | 18 ++++++++++++++++++ .../server/common/data/id/EntityIdTest.java | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java diff --git a/common/data/pom.xml b/common/data/pom.xml index 3b9152fb47..f8138928d6 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -119,6 +119,24 @@ false + + org.apache.maven.plugins + maven-surefire-plugin + ${surfire.version} + + + thingsboard + + + **/sql/*Test.java + **/nosql/*Test.java + + + **/*Test.java + **/*TestSuite.java + + + diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java new file mode 100644 index 0000000000..2aa67db36d --- /dev/null +++ b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java @@ -0,0 +1,13 @@ +package org.thingsboard.server.common.data.id; + +import org.junit.Assert; +import org.junit.Test; + +public class EntityIdTest { + + @Test + public void givenConstantNullUuid_whenCompare_thenToStringEqualsPredefinedUuid() { + Assert.assertEquals("13814000-1dd2-11b2-8080-808080808080", EntityId.NULL_UUID.toString()); + } + +} \ No newline at end of file From 7f3d63512e49932a2b2c5462dd44a9ae01e10007 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 27 Aug 2021 17:22:09 +0300 Subject: [PATCH 07/34] tests: added license header --- .../server/common/data/id/EntityIdTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java index 2aa67db36d..a53124860c 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/id/EntityIdTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.common.data.id; import org.junit.Assert; From a5d446914cae8fcb64ce5b0683285272d7cf2ec0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 30 Aug 2021 11:58:29 +0300 Subject: [PATCH 08/34] tests: relation query - added test cases with many edges between two nodes, horizontal ring (loop) on the same level, loop Tenant-Asset-Device-Tenant. This commit will fail 4 hierarchy tests. Reformat code --- .../dao/service/BaseEntityServiceTest.java | 70 ++++++++++++++----- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 831d92965b..2bd11f0914 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -82,6 +83,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; +import static org.hamcrest.MatcherAssert.assertThat; public abstract class BaseEntityServiceTest extends AbstractServiceTest { @@ -110,7 +112,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { tenantService.deleteTenant(tenantId); } - + @Test public void testCountEntitiesByQuery() throws InterruptedException { List devices = new ArrayList<>(); @@ -159,7 +161,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>()); RelationsQueryFilter filter = new RelationsQueryFilter(); filter.setRootEntity(tenantId); @@ -308,7 +310,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); List highTemperatures = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { @@ -373,14 +375,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - + @Test public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); List highTemperatures = new ArrayList<>(); - createTestHierarchy(assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); + createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { @@ -446,14 +448,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - + @Test public void testHierarchicalFindAssetsWithAttributesByQuery() throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List consumptions = new ArrayList<>(); List highConsumptions = new ArrayList<>(); - createTestHierarchy(assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); + createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { @@ -518,7 +520,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - private void createTestHierarchy(List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { + private void createTestHierarchy(TenantId tenantId, List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { for (int i = 0; i < 5; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); @@ -563,9 +565,41 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } } + + createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); + createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); + createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); + createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); + createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); + } + + void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { + assertThat("ids lenght", ids.length, Matchers.greaterThanOrEqualTo(1)); + //chain all from the head to the tail + for (int i = 1; i < ids.length; i++) { + relationService.saveRelation(tenantId, new EntityRelation(ids[i - 1], ids[i], type, RelationTypeGroup.COMMON)); + } + //chain tail -> head + relationService.saveRelation(tenantId, new EntityRelation(ids[ids.length - 1], ids[0], type, RelationTypeGroup.COMMON)); + } + + void createHorizontalRingRelations(TenantId tenantId, String type, List assets) { + createLoopRelations(tenantId, type, assets.stream().map(Asset::getId).toArray(EntityId[]::new)); + } + + void createManyCustomRelationsBetweenTwoNodes(TenantId tenantId, String type, List assets, List devices) { + for (int i = 1; i <= 5; i++) { + final String typeI = type + i; + createOneToManyRelations(tenantId, typeI, tenantId, assets.stream().map(Asset::getId).collect(Collectors.toList())); + assets.forEach(asset -> + createOneToManyRelations(tenantId, typeI, asset.getId(), devices.stream().map(Device::getId).collect(Collectors.toList()))); + } + } + + void createOneToManyRelations(TenantId tenantId, String type, EntityId from, List toIds) { + toIds.forEach(toId -> relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON))); } - @Test public void testSimpleFindEntityDataByQuery() throws InterruptedException { List devices = new ArrayList<>(); @@ -871,7 +905,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException{ + public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); @@ -1031,7 +1065,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - + @Test public void testFindEntityDataByQueryWithTimeseries() throws ExecutionException, InterruptedException { @@ -1122,7 +1156,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException{ + public void testBuildStringPredicateQueryOperations() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); List attributeStrings = new ArrayList<>(); @@ -1142,11 +1176,11 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { devices.add(deviceService.saveDevice(device)); //TO make sure devices have different created time Thread.sleep(1); - List operationValues= Arrays.asList(StringFilterPredicate.StringOperation.values()); + List operationValues = Arrays.asList(StringFilterPredicate.StringOperation.values()); StringFilterPredicate.StringOperation operation = operationValues.get(new Random().nextInt(operationValues.size())); String operationName = operation.name(); attributeStrings.add(operationName); - switch(operation){ + switch (operation) { case EQUAL: equalStrings.add(operationName); notContainsStrings.add(operationName); @@ -1302,7 +1336,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildStringPredicateQueryOperationsForEntityType() throws ExecutionException, InterruptedException{ + public void testBuildStringPredicateQueryOperationsForEntityType() throws ExecutionException, InterruptedException { List devices = new ArrayList<>(); @@ -1419,7 +1453,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } @Test - public void testBuildSimplePredicateQueryOperations() throws InterruptedException{ + public void testBuildSimplePredicateQueryOperations() throws InterruptedException { List devices = new ArrayList<>(); @@ -1492,7 +1526,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return loadedEntities; } - private List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value){ + private List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { KeyFilter filter = new KeyFilter(); filter.setKey(new EntityKey(keyType, key)); StringFilterPredicate predicate = new StringFilterPredicate(); @@ -1503,7 +1537,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return Collections.singletonList(filter); } - private KeyFilter createNumericKeyFilter(String key, EntityKeyType keyType, NumericFilterPredicate.NumericOperation operation, double value){ + private KeyFilter createNumericKeyFilter(String key, EntityKeyType keyType, NumericFilterPredicate.NumericOperation operation, double value) { KeyFilter filter = new KeyFilter(); filter.setKey(new EntityKey(keyType, key)); NumericFilterPredicate predicate = new NumericFilterPredicate(); From 891024f62011d8d491b67d96479d3d8c01dcb1ce Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 1 Sep 2021 14:59:49 +0300 Subject: [PATCH 09/34] relation-query-improvement, failed relation were disabled temporary and will be enabled on next commits --- .../query/DefaultEntityQueryRepository.java | 23 ++++++++++++++----- .../dao/service/BaseEntityServiceTest.java | 6 ++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 7b3ea36cfa..d47622cb8c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -239,18 +239,29 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ EntityType.TENANT, EntityType.CUSTOMER, EntityType.USER, EntityType.DASHBOARD, EntityType.ASSET, EntityType.DEVICE, EntityType.ENTITY_VIEW}; - private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, relation_type, lvl) AS (" + - " SELECT from_id, from_type, to_id, to_type, relation_type, 1 as lvl" + + private static final String HIERARCHICAL_QUERY_TEMPLATE = " FROM (WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) AS (" + + " SELECT from_id, from_type, to_id, to_type," + + " 1 as lvl," + + " ARRAY[$in_id] as path" + // initial path " FROM relation" + " WHERE $in_id = :relation_root_id and $in_type = :relation_root_type and relation_type_group = 'COMMON'" + " UNION ALL" + - " SELECT r.from_id, r.from_type, r.to_id, r.to_type, r.relation_type, lvl + 1" + + " SELECT r.from_id, r.from_type, r.to_id, r.to_type," + + " (re.lvl + 1) as lvl, " + + " (re.path || ARRAY[r.$in_id]) as path" + " FROM relation r" + " INNER JOIN related_entities re ON" + " r.$in_id = re.$out_id and r.$in_type = re.$out_type and" + - " relation_type_group = 'COMMON' %s)" + - " SELECT re.$out_id entity_id, re.$out_type entity_type, max(re.lvl) lvl" + - " from related_entities re" + + " relation_type_group = 'COMMON' " + + " AND r.$in_id NOT IN (SELECT * FROM unnest(re.path)) " + + " %s" + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, (re.lvl + 1), (re.path || ARRAY[r.$in_id])" + + " )" + + " SELECT re.$out_id entity_id, re.$out_type entity_type, max(r_int.lvl) lvl" + + " from related_entities r_int" + + " INNER JOIN relation re ON re.from_id = r_int.from_id AND re.from_type = r_int.from_type" + + " AND re.to_id = r_int.to_id AND re.to_type = r_int.to_type" + + " AND re.relation_type_group = 'COMMON'" + " %s GROUP BY entity_id, entity_type) entity"; private static final String HIERARCHICAL_TO_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "to").replace("$out", "from"); private static final String HIERARCHICAL_FROM_QUERY_TEMPLATE = HIERARCHICAL_QUERY_TEMPLATE.replace("$in", "from").replace("$out", "to"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 2bd11f0914..68632802b6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -567,10 +567,10 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); - createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); + //createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); - createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); - createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); + //createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); + //createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); } void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { From 2d2bd2c1586aed1012aae8dd7ccc43dbade0ca8e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 3 Sep 2021 08:54:00 +0300 Subject: [PATCH 10/34] reverted unnecessary declaration for maven-surefire-plugin on common/data where only simple test present (will run by default) --- common/data/pom.xml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/common/data/pom.xml b/common/data/pom.xml index f8138928d6..3b9152fb47 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -119,24 +119,6 @@ false - - org.apache.maven.plugins - maven-surefire-plugin - ${surfire.version} - - - thingsboard - - - **/sql/*Test.java - **/nosql/*Test.java - - - **/*Test.java - **/*TestSuite.java - - - From 9ceb6f0d4d635d6efdb4a944b4b9cdcae96b575e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 3 Sep 2021 15:15:15 +0300 Subject: [PATCH 11/34] relational query: stable test run with ManyCustomRelationsBetweenTwoNodes, but without Loop Relations --- .../thingsboard/server/dao/service/BaseEntityServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 68632802b6..1eea71a408 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -568,7 +568,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); //createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); - createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); + //createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); //createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); //createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); } From 99476c78c9fc165ea00970f25d27001df4a735b0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 3 Sep 2021 15:33:40 +0300 Subject: [PATCH 12/34] relational query failed test when createHorizontalRingRelations added and freeze when createLoopRelations (due to infinite recursion with no level limit) --- .../thingsboard/server/dao/service/BaseEntityServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 1eea71a408..877bf366b5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -567,7 +567,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); - //createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); + createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); //createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); //createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); //createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); From e4f8c7e8788d5551a4d2c8b296b884d2fb337756 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 3 Sep 2021 16:05:14 +0300 Subject: [PATCH 13/34] relational query: Limit the max level to break the infinite loop when circles appear in relation graph --- .../query/DefaultEntityQueryRepository.java | 9 +++- .../dao/service/BaseEntityServiceTest.java | 6 +-- .../DefaultEntityQueryRepositoryTest.java | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index d47622cb8c..e1c3a83936 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -223,6 +223,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private static final String SELECT_API_USAGE_STATE = "(select aus.id, aus.created_time, aus.tenant_id, aus.entity_id, " + "coalesce((select title from tenant where id = aus.entity_id), (select title from customer where id = aus.entity_id)) as name " + "from api_usage_state as aus)"; + static final int MAX_LEVEL_DEFAULT = 10; //This value has to be reasonable small to prevent infinite recursion as early as possible static { entityTableMap.put(EntityType.ASSET, "asset"); @@ -704,8 +705,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return whereFilter.toString(); } - private String getLvlFilter(int maxLevel) { - return maxLevel > 0 ? ("and lvl <= " + (maxLevel - 1)) : ""; + String getLvlFilter(int maxLevel) { + return "and re.lvl <= " + (getMaxLevel(maxLevel) - 1); + } + + int getMaxLevel(int maxLevel) { + return maxLevel > 0 ? maxLevel : MAX_LEVEL_DEFAULT; } private String getQueryTemplate(EntitySearchDirection direction) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 877bf366b5..2bd11f0914 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -568,9 +568,9 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); - //createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); - //createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); - //createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); + createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); + createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); + createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); } void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java new file mode 100644 index 0000000000..6e67a4946f --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.dao.sql.query; + +import org.junit.Test; +import org.thingsboard.server.common.data.id.CustomerId; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.Mockito.mock; + +public class DefaultEntityQueryRepositoryTest { + + /* + * This value has to be reasonable small to prevent infinite recursion as early as possible + * */ + @Test + public void givenDefaultMaxLevel_whenStaticConstant_thenEqualsTo() { + assertThat(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT, equalTo(10)); + } + + @Test + public void givenMaxLevelZeroOrNegative_whenGetMaxLevel_thenReturnDefaultMaxLevel() { + DefaultEntityQueryRepository repo = mock(DefaultEntityQueryRepository.class); + willCallRealMethod().given(repo).getMaxLevel(anyInt()); + assertThat(repo.getMaxLevel(0), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(-1), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(-2), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + assertThat(repo.getMaxLevel(Integer.MIN_VALUE), equalTo(DefaultEntityQueryRepository.MAX_LEVEL_DEFAULT)); + } + + @Test + public void givenMaxLevelPositive_whenGetMaxLevel_thenValueTheSame() { + DefaultEntityQueryRepository repo = mock(DefaultEntityQueryRepository.class); + willCallRealMethod().given(repo).getMaxLevel(anyInt()); + assertThat(repo.getMaxLevel(1), equalTo(1)); + assertThat(repo.getMaxLevel(2), equalTo(2)); + assertThat(repo.getMaxLevel(Integer.MAX_VALUE), equalTo(Integer.MAX_VALUE)); + } + +} From d65b02df19da10c97ca4c11aea1910e5721e89e5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 6 Sep 2021 11:18:06 +0300 Subject: [PATCH 14/34] test: count entities by query adjusted due to the loop relations in hierarchy --- .../thingsboard/server/dao/service/BaseEntityServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 2bd11f0914..f8dd6cc344 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -170,7 +170,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { EntityCountQuery countQuery = new EntityCountQuery(filter); long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(30, count); + Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); From f2f34680779cbf6aab7af1600a698e85e45eeb3f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 6 Sep 2021 13:00:34 +0300 Subject: [PATCH 15/34] test relations: print all relations added as sql procedure to reproduce exact data in the PostgreSQL --- .../dao/service/BaseEntityServiceTest.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index f8dd6cc344..357af4e644 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -70,6 +70,7 @@ import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; +import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.ArrayList; @@ -98,6 +99,9 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { @Autowired private JdbcTemplate template; + @Autowired + private RelationRepository relationRepository; + @Before public void before() { Tenant tenant = new Tenant(); @@ -156,7 +160,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { Assert.assertEquals(0, count); } - + @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); @@ -571,6 +575,47 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); + + printAllRelations(); + } + + /* + * This useful to reproduce exact data in the PostgreSQL and play around with pgadmin query and analyze tool + * */ + private void printAllRelations() { + System.out.println("" + + "DO\n" + + "$$\n" + + " DECLARE\n" + + " someint integer;\n" + + " BEGIN\n" + + " DROP TABLE IF EXISTS relation_test;\n" + + " CREATE TABLE IF NOT EXISTS relation_test\n" + + " (\n" + + " from_id uuid,\n" + + " from_type varchar(255),\n" + + " to_id uuid,\n" + + " to_type varchar(255),\n" + + " relation_type_group varchar(255),\n" + + " relation_type varchar(255),\n" + + " additional_info varchar,\n" + + " CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" + + " );"); + + relationRepository.findAll().forEach(r -> + System.out.printf("INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (%s, %s, %s, %s, %s, %s, %s);\n", + quote(r.getFromId()), quote(r.getFromType()), quote(r.getToId()), quote(r.getToType()), + quote(r.getRelationTypeGroup()), quote(r.getRelationType()), quote(r.getAdditionalInfo())) + ); + + System.out.println("" + + " END\n" + + "$$;"); + } + + private String quote(Object s) { + return s == null ? null : "'" + s + "'"; } void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { From edd96624ae6868ab1f06ee382cbabb8e128fcffb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 7 Sep 2021 16:26:25 +0300 Subject: [PATCH 16/34] relation query: group by added to the initial pert of recursive query --- .../server/dao/sql/query/DefaultEntityQueryRepository.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index e1c3a83936..22d2ac62f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -223,7 +223,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private static final String SELECT_API_USAGE_STATE = "(select aus.id, aus.created_time, aus.tenant_id, aus.entity_id, " + "coalesce((select title from tenant where id = aus.entity_id), (select title from customer where id = aus.entity_id)) as name " + "from api_usage_state as aus)"; - static final int MAX_LEVEL_DEFAULT = 10; //This value has to be reasonable small to prevent infinite recursion as early as possible + static final int MAX_LEVEL_DEFAULT = 50; //This value has to be reasonable small to prevent infinite recursion as early as possible static { entityTableMap.put(EntityType.ASSET, "asset"); @@ -244,8 +244,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { " SELECT from_id, from_type, to_id, to_type," + " 1 as lvl," + " ARRAY[$in_id] as path" + // initial path - " FROM relation" + + " FROM relation " + " WHERE $in_id = :relation_root_id and $in_type = :relation_root_type and relation_type_group = 'COMMON'" + + " GROUP BY from_id, from_type, to_id, to_type, lvl, path" + " UNION ALL" + " SELECT r.from_id, r.from_type, r.to_id, r.to_type," + " (re.lvl + 1) as lvl, " + From 1adaef31f529c82f34349215d5919634ba85f1c8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 12 Oct 2021 18:28:44 +0300 Subject: [PATCH 17/34] license header CE fixed for DefaultEntityQueryRepositoryTest --- .../query/DefaultEntityQueryRepositoryTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java index 6e67a4946f..a0c716d51f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepositoryTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.sql.query; import org.junit.Test; From adb9a8d1501bf37e3ba918ef2b8a5617508555cd Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 7 Sep 2021 20:23:24 +0300 Subject: [PATCH 18/34] dao tests: PostgreSqlDaoServiceTestSuite added using org.testcontainers --- dao/pom.xml | 15 ++ .../dao/PostgreSqlDaoServiceTestSuite.java | 76 ++++++++ .../dao/service/BaseEntityServiceTest.java | 177 ++++++++++++++++-- .../server/dao/service/DaoPostgreSqlTest.java | 48 +++++ .../EntityServicePostgreSqlTest.java} | 8 +- dao/src/test/resources/psql-test.properties | 47 +++++ .../test/resources/sql/system-test-psql.sql | 2 + pom.xml | 11 ++ 8 files changed, 363 insertions(+), 21 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java rename dao/src/test/java/org/thingsboard/server/dao/service/{sql/EntityServiceSqlTest.java => psql/EntityServicePostgreSqlTest.java} (77%) create mode 100644 dao/src/test/resources/psql-test.properties create mode 100644 dao/src/test/resources/sql/system-test-psql.sql diff --git a/dao/pom.xml b/dao/pom.xml index 922b389bff..1488d7eec5 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -201,6 +201,11 @@ org.springframework.boot spring-boot-starter-data-jpa + + org.springframework.boot + spring-boot-starter-test + test + org.springframework spring-test @@ -211,6 +216,16 @@ hsqldb test + + org.testcontainers + postgresql + test + + + org.testcontainers + jdbc + test + org.springframework spring-context-support diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java new file mode 100644 index 0000000000..5d6f1a374f --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java @@ -0,0 +1,76 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao; + +import org.junit.ClassRule; +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; + +import java.util.Arrays; + +@RunWith(ClasspathSuite.class) +@ClassnameFilters({ + "org.thingsboard.server.dao.service.psql.*SqlTest", + "org.thingsboard.server.dao.service.attributes.psql.*SqlTest", + "org.thingsboard.server.dao.service.event.psql.*SqlTest", + "org.thingsboard.server.dao.service.timeseries.psql.*SqlTest" +}) +public class PostgreSqlDaoServiceTestSuite { + + @ClassRule + public static CustomSqlUnit sqlUnit = new CustomSqlUnit( + Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql" + , "sql/system-data.sql" + , "sql/system-test-psql.sql" + ), + "sql/psql/drop-all-tables.sql", + "psql-test.properties" + ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-ts-psql.sql" +// , "sql/schema-entities.sql", "sql/schema-entities-idx.sql" +// , "sql/system-data.sql", "sql/system-test.sql" +// ), +// "sql/psql/drop-all-tables.sql", +// "sql-test.properties" +// ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), +// "sql/timescale/drop-all-tables.sql", +// "sql-test.properties" +// ); + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 357af4e644..fbace01ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -17,15 +17,18 @@ package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.hamcrest.Matchers; +import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -80,14 +83,17 @@ import java.util.Comparator; import java.util.List; import java.util.Random; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.hamcrest.MatcherAssert.assertThat; +@Slf4j public abstract class BaseEntityServiceTest extends AbstractServiceTest { + static final int ENTITY_COUNT = 5; @Autowired private AttributesService attributesService; @@ -525,7 +531,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } private void createTestHierarchy(TenantId tenantId, List assets, List devices, List consumptions, List highConsumptions, List temperatures, List highTemperatures) throws InterruptedException { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < ENTITY_COUNT; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("Asset" + i); @@ -535,18 +541,19 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); assets.add(asset); - EntityRelation er = new EntityRelation(); - er.setFrom(tenantId); - er.setTo(asset.getId()); - er.setType("Manages"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Manages", tenantId, asset.getId()); long consumption = (long) (Math.random() * 100); consumptions.add(consumption); if (consumption > 50) { highConsumptions.add(consumption); } - for (int j = 0; j < 5; j++) { + + //tenant -> asset : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, tenantId, asset.getId()); + } + + for (int j = 0; j < ENTITY_COUNT; j++) { Device device = new Device(); device.setTenantId(tenantId); device.setName("A" + i + "Device" + j); @@ -556,27 +563,158 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { //TO make sure devices have different created time Thread.sleep(1); devices.add(device); - er = new EntityRelation(); - er.setFrom(asset.getId()); - er.setTo(device.getId()); - er.setType("Contains"); - er.setTypeGroup(RelationTypeGroup.COMMON); - relationService.saveRelation(tenantId, er); + createRelation(tenantId, "Contains", asset.getId(), device.getId()); long temperature = (long) (Math.random() * 100); temperatures.add(temperature); if (temperature > 45) { highTemperatures.add(temperature); } + + //asset -> device : one-to-one but many edges + for (int n = 0; n < ENTITY_COUNT; n++) { + createRelation(tenantId, "UseCase-" + n, asset.getId(), device.getId()); + } } } - createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); + //asset -> device one-to-many shared with other assets + for (int n = 0; n < devices.size(); n = n + ENTITY_COUNT) { + createRelation(tenantId, "SharedWithAsset0", assets.get(0).getId(), devices.get(n).getId()); + } + + //createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); - printAllRelations(); + //testQuery(); + + //printAllRelations(); + } + + private void testQuery() { + + template.query("" + + " DROP TABLE IF EXISTS relation_test;\n" + + " CREATE TABLE IF NOT EXISTS relation_test\n" + + " (\n" + + " from_id uuid,\n" + + " from_type varchar(255),\n" + + " to_id uuid,\n" + + " to_type varchar(255),\n" + + " relation_type_group varchar(255),\n" + + " relation_type varchar(255),\n" + + " additional_info varchar,\n" + + " CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" + + " );\n", r -> null); + + log.error("insert test {}", (Object) template.query("" + + "INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " + + " VALUES " + + " ('11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', '22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', 'COMMON', 'Contains', null),\n" + + " ('22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', '33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', 'COMMON', 'Contains', null),\n" + + " ('33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', '11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', 'COMMON', 'Contains', null);\n" + + "" + , r -> null)); + + + //log.error("array_position hsql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue','wed','thu','fri','sat'], 'mon') from relation_test", Long.class)); + + log.error("array_position hsql {}", template.queryForObject("select position_array('mon' in ARRAY['sun','mon','tue']);", Long.class)); + log.error("array_position psql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue'], 'mon');", Long.class)); + + Long countTest = template.queryForObject("select count(*) from relation_test", Long.class); + log.error("count test {}", countTest); + + Long count = template.queryForObject("select count(*) from relation", Long.class); + log.error("count {}", count); + + List> result = template.query("" + + "WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) " + + " AS (SELECT from_id, " + + " from_type, " + + " to_id, " + + " to_type, " + + " 1 as lvl, " + + " ARRAY [from_id] as path " + + " FROM relation_test r " + + " WHERE from_id = '11111111-0f19-11ec-ba23-e981fc95500d' " + + " and from_type = 'TENANT' " + + " and relation_type_group = 'COMMON' " + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type , 1, ARRAY [from_id] " + + " UNION ALL " + + " SELECT r.from_id, " + + " r.from_type, " + + " r.to_id, " + + " r.to_type, " + + " (re.lvl + 1) as lvl, " + + " (re.path || ARRAY [r.from_id]) as path " + + " FROM relation_test r " + + " INNER JOIN related_entities re " + + " ON r.from_id = re.to_id and " + + " r.from_type = re.to_type and " + + " relation_type_group = 'COMMON' and " + + " r.from_id NOT IN (SELECT * FROM unnest(re.path)) and " + + " re.lvl <= 7 " + + " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, " + + " (re.lvl + 1), (re.path || ARRAY [r.from_id])) " + + " " + + " SELECT lvl, from_id, from_type, to_id, to_type, path " + //to_id IN (SELECT * FROM unnest(path)) as is_present, + " from related_entities r_int " + + " " + + " ", + getListResultSetExtractor()); + + log.error("result {}", result.size() - 1); + AtomicInteger counter = new AtomicInteger(); + result.forEach(r -> System.out.printf("%s %s\n", counter.incrementAndGet(), r.toString())); + log.error("end"); + + log.error("query 1 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')])) ", + String.class)); + log.error("query 1.1 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')] || ARRAY[UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')] )) ", + String.class)); + log.error("query 2 (expected true): {}", template.queryForObject( + "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", + String.class)); +// log.error("query true3: ", template.queryForObject( +// "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", +// String.class)); + log.error("query 3 (expected true): {} ", template.queryForObject( + "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' NOT IN (SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3') ", + String.class)); + + List> result2 = template.query("SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')]) " + , getListResultSetExtractor()); + log.error("result2 {}", result2.size() - 1); + AtomicInteger counter2 = new AtomicInteger(); + result2.forEach(r -> System.out.printf("%s %s\n", counter2.incrementAndGet(), r.toString())); + log.error("end2"); + + } + + @NotNull + private ResultSetExtractor>> getListResultSetExtractor() { + return rs -> { + List> list = new ArrayList<>(); + final int columnCount = rs.getMetaData().getColumnCount(); + List columns = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + columns.add(rs.getMetaData().getColumnName(i)); + } + list.add(columns); + while (rs.next()) { + List data = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + data.add(rs.getString(i)); + } + list.add(data); + } + return list; + }; } /* @@ -642,9 +780,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } void createOneToManyRelations(TenantId tenantId, String type, EntityId from, List toIds) { - toIds.forEach(toId -> relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON))); + toIds.forEach(toId -> createRelation(tenantId, type, from, toId)); + } + + void createRelation(TenantId tenantId, String type, EntityId from, EntityId toId) { + relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON)); } + @Test public void testSimpleFindEntityDataByQuery() throws InterruptedException { List devices = new ArrayList<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java new file mode 100644 index 0000000000..16da531369 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java @@ -0,0 +1,48 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao.service; + +import org.springframework.test.context.TestPropertySource; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@TestPropertySource(locations = {"classpath:application-test.properties", "classpath:psql-test.properties"}) +public @interface DaoPostgreSqlTest { +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java similarity index 77% rename from dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java index 0c59ae49c8..5b8c6d4aca 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service.sql; +package org.thingsboard.server.dao.service.psql; import org.thingsboard.server.dao.service.BaseEntityServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.service.DaoPostgreSqlTest; -@DaoSqlTest -public class EntityServiceSqlTest extends BaseEntityServiceTest { +@DaoPostgreSqlTest +public class EntityServicePostgreSqlTest extends BaseEntityServiceTest { } diff --git a/dao/src/test/resources/psql-test.properties b/dao/src/test/resources/psql-test.properties new file mode 100644 index 0000000000..00420be628 --- /dev/null +++ b/dao/src/test/resources/psql-test.properties @@ -0,0 +1,47 @@ +database.ts.type=sql +database.ts_latest.type=sql +sql.ts_inserts_executor_type=fixed +sql.ts_inserts_fixed_thread_pool_size=200 +sql.ts_key_value_partitioning=MONTHS +# +spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +spring.jpa.properties.hibernate.order_by.default_null_ordering=last +spring.jpa.properties.hibernate.jdbc.log.warnings=false +spring.jpa.show-sql=false +spring.jpa.hibernate.ddl-auto=none +spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +spring.datasource.username=postgres +spring.datasource.password=postgres +spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw +spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver +#org.postgresql.Driver +spring.datasource.hikari.maximumPoolSize=20 +service.type=monolith +#database.ts.type=timescale +#database.ts.type=sql +#database.entities.type=sql +# +#sql.ts_inserts_executor_type=fixed +#sql.ts_inserts_fixed_thread_pool_size=200 +#sql.ts_key_value_partitioning=MONTHS +# +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +#spring.jpa.show-sql=false +#spring.jpa.hibernate.ddl-auto=none +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +# +#spring.datasource.username=postgres +#spring.datasource.password=postgres +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest +#spring.datasource.driverClassName=org.postgresql.Driver +#spring.datasource.hikari.maximumPoolSize = 50 +queue.core.pack-processing-timeout=3000 +queue.rule-engine.pack-processing-timeout=3000 +queue.rule-engine.queues[0].name=Main +queue.rule-engine.queues[0].topic=tb_rule_engine.main +queue.rule-engine.queues[0].poll-interval=25 +queue.rule-engine.queues[0].partitions=3 +queue.rule-engine.queues[0].pack-processing-timeout=3000 +queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES +queue.rule-engine.queues[0].submit-strategy.type=BURST +sql.log_entity_queries=true diff --git a/dao/src/test/resources/sql/system-test-psql.sql b/dao/src/test/resources/sql/system-test-psql.sql new file mode 100644 index 0000000000..16dcb8c3ae --- /dev/null +++ b/dao/src/test/resources/sql/system-test-psql.sql @@ -0,0 +1,2 @@ +--PostgreSQL specific truncate to fit constraints +TRUNCATE TABLE device_credentials, device, device_profile, rule_node_state, rule_node, rule_chain; \ No newline at end of file diff --git a/pom.xml b/pom.xml index fa33548a56..2cc0f0d6b1 100755 --- a/pom.xml +++ b/pom.xml @@ -1637,6 +1637,17 @@ org.hsqldb hsqldb ${hsqldb.version} + + + org.testcontainers + postgresql + ${testcontainers.version} + test + + + org.testcontainers + jdbc + ${testcontainers.version} test From 076b1943fb974d4093275ff54fca4337b0ca88a6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 8 Sep 2021 10:12:25 +0300 Subject: [PATCH 19/34] sql dao test: init ODBC connection to Postgres DB with custom PostgreSqlInitializer (class rule replacement, PostgreSQL dao tests able to run as standalone) --- .../dao/PostgreSqlDaoServiceTestSuite.java | 31 -------- .../server/dao/PostgreSqlInitializer.java | 79 +++++++++++++++++++ dao/src/test/resources/psql-test.properties | 4 +- 3 files changed, 81 insertions(+), 33 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java index 5d6f1a374f..915379fe74 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java @@ -30,13 +30,10 @@ */ package org.thingsboard.server.dao; -import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; import org.junit.runner.RunWith; -import java.util.Arrays; - @RunWith(ClasspathSuite.class) @ClassnameFilters({ "org.thingsboard.server.dao.service.psql.*SqlTest", @@ -45,32 +42,4 @@ import java.util.Arrays; "org.thingsboard.server.dao.service.timeseries.psql.*SqlTest" }) public class PostgreSqlDaoServiceTestSuite { - - @ClassRule - public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql" - , "sql/system-data.sql" - , "sql/system-test-psql.sql" - ), - "sql/psql/drop-all-tables.sql", - "psql-test.properties" - ); - -// @ClassRule -// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( -// Arrays.asList("sql/schema-ts-psql.sql" -// , "sql/schema-entities.sql", "sql/schema-entities-idx.sql" -// , "sql/system-data.sql", "sql/system-test.sql" -// ), -// "sql/psql/drop-all-tables.sql", -// "sql-test.properties" -// ); - -// @ClassRule -// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( -// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), -// "sql/timescale/drop-all-tables.sql", -// "sql-test.properties" -// ); - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java new file mode 100644 index 0000000000..5f70b7ad44 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java @@ -0,0 +1,79 @@ +/** + * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * + * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of ThingsBoard, Inc. and its suppliers, + * if any. The intellectual and technical concepts contained + * herein are proprietary to ThingsBoard, Inc. + * and its suppliers and may be covered by U.S. and Foreign Patents, + * patents in process, and are protected by trade secret or copyright law. + * + * Dissemination of this information or reproduction of this material is strictly forbidden + * unless prior written permission is obtained from COMPANY. + * + * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, + * managers or contractors who have executed Confidentiality and Non-disclosure agreements + * explicitly covering such access. + * + * The copyright notice above does not evidence any actual or intended publication + * or disclosure of this source code, which includes + * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. + * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, + * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT + * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, + * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. + * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION + * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, + * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + */ +package org.thingsboard.server.dao; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +@Slf4j +public class PostgreSqlInitializer { + + private static final List sqlFiles = List.of( + "sql/schema-ts-psql.sql", + "sql/schema-entities.sql", + "sql/schema-entities-idx.sql", + "sql/system-data.sql", + "sql/system-test-psql.sql"); + private static final String dropAllTablesSqlFile = "sql/psql/drop-all-tables.sql"; + + public static void initDb(Connection conn) { + cleanUpDb(conn); + log.info("initialize Postgres DB..."); + try { + for (String sqlFile : sqlFiles) { + URL sqlFileUrl = Resources.getResource(sqlFile); + String sql = Resources.toString(sqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(sql); + } + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to init the Postgres database. Reason: " + e.getMessage(), e); + } + log.info("Postgres DB is initialized!"); + } + + private static void cleanUpDb(Connection conn) { + log.info("clean up Postgres DB..."); + try { + URL dropAllTableSqlFileUrl = Resources.getResource(dropAllTablesSqlFile); + String dropAllTablesSql = Resources.toString(dropAllTableSqlFileUrl, Charsets.UTF_8); + conn.createStatement().execute(dropAllTablesSql); + } catch (IOException | SQLException e) { + throw new RuntimeException("Unable to clean up the Postgres database. Reason: " + e.getMessage(), e); + } + } +} diff --git a/dao/src/test/resources/psql-test.properties b/dao/src/test/resources/psql-test.properties index 00420be628..fb65966acf 100644 --- a/dao/src/test/resources/psql-test.properties +++ b/dao/src/test/resources/psql-test.properties @@ -12,10 +12,10 @@ spring.jpa.hibernate.ddl-auto=none spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect spring.datasource.username=postgres spring.datasource.password=postgres -spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw +spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver #org.postgresql.Driver -spring.datasource.hikari.maximumPoolSize=20 +spring.datasource.hikari.maximumPoolSize=50 service.type=monolith #database.ts.type=timescale #database.ts.type=sql From 7585239339c7dc72f6a2bc3a377fe18c27fbaec0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 8 Sep 2021 10:24:18 +0300 Subject: [PATCH 20/34] tests: cleanup after dubug --- .../dao/service/BaseEntityServiceTest.java | 110 +----------------- 1 file changed, 1 insertion(+), 109 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index fbace01ddb..cd4763e4c0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.hamcrest.Matchers; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -83,7 +82,6 @@ import java.util.Comparator; import java.util.List; import java.util.Random; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -582,121 +580,15 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { createRelation(tenantId, "SharedWithAsset0", assets.get(0).getId(), devices.get(n).getId()); } - //createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); + createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices); createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets); createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId()); createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId()); createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId()); - //testQuery(); - //printAllRelations(); } - private void testQuery() { - - template.query("" + - " DROP TABLE IF EXISTS relation_test;\n" + - " CREATE TABLE IF NOT EXISTS relation_test\n" + - " (\n" + - " from_id uuid,\n" + - " from_type varchar(255),\n" + - " to_id uuid,\n" + - " to_type varchar(255),\n" + - " relation_type_group varchar(255),\n" + - " relation_type varchar(255),\n" + - " additional_info varchar,\n" + - " CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" + - " );\n", r -> null); - - log.error("insert test {}", (Object) template.query("" + - "INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " + - " VALUES " + - " ('11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', '22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', 'COMMON', 'Contains', null),\n" + - " ('22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', '33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', 'COMMON', 'Contains', null),\n" + - " ('33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', '11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', 'COMMON', 'Contains', null);\n" + - "" - , r -> null)); - - - //log.error("array_position hsql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue','wed','thu','fri','sat'], 'mon') from relation_test", Long.class)); - - log.error("array_position hsql {}", template.queryForObject("select position_array('mon' in ARRAY['sun','mon','tue']);", Long.class)); - log.error("array_position psql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue'], 'mon');", Long.class)); - - Long countTest = template.queryForObject("select count(*) from relation_test", Long.class); - log.error("count test {}", countTest); - - Long count = template.queryForObject("select count(*) from relation", Long.class); - log.error("count {}", count); - - List> result = template.query("" + - "WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) " + - " AS (SELECT from_id, " + - " from_type, " + - " to_id, " + - " to_type, " + - " 1 as lvl, " + - " ARRAY [from_id] as path " + - " FROM relation_test r " + - " WHERE from_id = '11111111-0f19-11ec-ba23-e981fc95500d' " + - " and from_type = 'TENANT' " + - " and relation_type_group = 'COMMON' " + - " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type , 1, ARRAY [from_id] " + - " UNION ALL " + - " SELECT r.from_id, " + - " r.from_type, " + - " r.to_id, " + - " r.to_type, " + - " (re.lvl + 1) as lvl, " + - " (re.path || ARRAY [r.from_id]) as path " + - " FROM relation_test r " + - " INNER JOIN related_entities re " + - " ON r.from_id = re.to_id and " + - " r.from_type = re.to_type and " + - " relation_type_group = 'COMMON' and " + - " r.from_id NOT IN (SELECT * FROM unnest(re.path)) and " + - " re.lvl <= 7 " + - " GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, " + - " (re.lvl + 1), (re.path || ARRAY [r.from_id])) " + - " " + - " SELECT lvl, from_id, from_type, to_id, to_type, path " + //to_id IN (SELECT * FROM unnest(path)) as is_present, - " from related_entities r_int " + - " " + - " ", - getListResultSetExtractor()); - - log.error("result {}", result.size() - 1); - AtomicInteger counter = new AtomicInteger(); - result.forEach(r -> System.out.printf("%s %s\n", counter.incrementAndGet(), r.toString())); - log.error("end"); - - log.error("query 1 (expected true): {}", template.queryForObject( - "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')])) ", - String.class)); - log.error("query 1.1 (expected true): {}", template.queryForObject( - "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')] || ARRAY[UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')] )) ", - String.class)); - log.error("query 2 (expected true): {}", template.queryForObject( - "SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", - String.class)); -// log.error("query true3: ", template.queryForObject( -// "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ", -// String.class)); - log.error("query 3 (expected true): {} ", template.queryForObject( - "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' NOT IN (SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3') ", - String.class)); - - List> result2 = template.query("SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')]) " - , getListResultSetExtractor()); - log.error("result2 {}", result2.size() - 1); - AtomicInteger counter2 = new AtomicInteger(); - result2.forEach(r -> System.out.printf("%s %s\n", counter2.incrementAndGet(), r.toString())); - log.error("end2"); - - } - - @NotNull private ResultSetExtractor>> getListResultSetExtractor() { return rs -> { List> list = new ArrayList<>(); From 2d2aa50520346ac2d302d1f334073e64db376171 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 8 Sep 2021 10:25:29 +0300 Subject: [PATCH 21/34] tests: added psql exclude in application pom. dao pom copied the same rules as fo rapplication --- application/pom.xml | 1 + dao/pom.xml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/application/pom.xml b/application/pom.xml index f454a462b9..3378d52459 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -360,6 +360,7 @@ **/sql/*Test.java + **/psql/*Test.java **/nosql/*Test.java diff --git a/dao/pom.xml b/dao/pom.xml index 1488d7eec5..2dd76cfcd4 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -254,7 +254,13 @@ maven-surefire-plugin ${surfire.version} + + **/sql/*Test.java + **/psql/*Test.java + **/nosql/*Test.java + + **/*Test.java **/*TestSuite.java From f4cfd92aaa265e7055d1fb634e58239889a5f6c0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 8 Sep 2021 10:47:38 +0300 Subject: [PATCH 22/34] tests dao: marked base classes as abstract, exclude subfolders for sql test on in pom --- dao/pom.xml | 1 + .../server/dao/service/BaseDeviceProfileServiceTest.java | 2 +- .../server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java | 2 +- .../thingsboard/server/dao/service/BaseOAuth2ServiceTest.java | 2 +- .../server/dao/service/BaseTenantProfileServiceTest.java | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dao/pom.xml b/dao/pom.xml index 2dd76cfcd4..f490a0e8e5 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -256,6 +256,7 @@ **/sql/*Test.java + **/sql/*/*Test.java **/psql/*Test.java **/nosql/*Test.java diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 2a1ff95062..cfd7923969 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -47,7 +47,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; -public class BaseDeviceProfileServiceTest extends AbstractServiceTest { +public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java index c3073434e0..743591863a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ConfigTemplateServiceTest.java @@ -31,7 +31,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import java.util.Arrays; import java.util.UUID; -public class BaseOAuth2ConfigTemplateServiceTest extends AbstractServiceTest { +public abstract class BaseOAuth2ConfigTemplateServiceTest extends AbstractServiceTest { @Autowired protected OAuth2ConfigTemplateService oAuth2ConfigTemplateService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java index 4077bb2176..0ad3470c99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOAuth2ServiceTest.java @@ -43,7 +43,7 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; -public class BaseOAuth2ServiceTest extends AbstractServiceTest { +public abstract class BaseOAuth2ServiceTest extends AbstractServiceTest { private static final OAuth2Info EMPTY_PARAMS = new OAuth2Info(false, Collections.emptyList()); @Autowired diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index 2ebb955c61..d77143f0a0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -34,7 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -public class BaseTenantProfileServiceTest extends AbstractServiceTest { +public abstract class BaseTenantProfileServiceTest extends AbstractServiceTest { private IdComparator idComparator = new IdComparator<>(); private IdComparator tenantProfileInfoIdComparator = new IdComparator<>(); From 8b6a9be476c76da0871c219ad96df48c59be2c03 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 10 Sep 2021 12:29:48 +0300 Subject: [PATCH 23/34] fixed re.lvl with r_int.lvl --- .../server/dao/sql/query/DefaultEntityQueryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 22d2ac62f2..f567420947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -593,7 +593,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append("nr.").append(fromOrTo).append("_type").append(" = re.").append(toOrFrom).append("_type"); notExistsPart.append(")"); - whereFilter += " and ( re.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; + whereFilter += " and ( r_int.lvl = " + entityFilter.getMaxLevel() + " OR " + notExistsPart.toString() + ")"; } from = String.format(from, lvlFilter, whereFilter); String query = "( " + selectFields + from + ")"; @@ -672,7 +672,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { .append(whereFilter.toString().replaceAll("re\\.", "nr\\.")); notExistsPart.append(")"); - whereFilter.append(" and ( re.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); + whereFilter.append(" and ( r_int.lvl = ").append(entityFilter.getMaxLevel()).append(" OR ").append(notExistsPart.toString()).append(")"); } from = String.format(from, lvlFilter, " WHERE " + whereFilter); return "( " + selectFields + from + ")"; From 2d08573feeff00949305d616e4afdd5046e2dcc5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 15 Sep 2021 11:00:45 +0300 Subject: [PATCH 24/34] relation query test: maxLevel and fetchLastLevelOnly cases added --- .../dao/service/BaseEntityServiceTest.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index cd4763e4c0..e58191b60d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -92,6 +92,7 @@ import static org.hamcrest.MatcherAssert.assertThat; public abstract class BaseEntityServiceTest extends AbstractServiceTest { static final int ENTITY_COUNT = 5; + @Autowired private AttributesService attributesService; @@ -314,6 +315,20 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { @Test public void testHierarchicalFindEntityDataWithAttributesByQuery() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(0, false); + } + + @Test + public void testHierarchicalFindEntityDataWithAttributesByQueryWithLevel() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(2, false); + } + + @Test + public void testHierarchicalFindEntityDataWithAttributesByQueryWithLastLevelOnly() throws ExecutionException, InterruptedException { + doTestHierarchicalFindEntityDataWithAttributesByQuery(2, true); + } + + private void doTestHierarchicalFindEntityDataWithAttributesByQuery(final int maxLevel, final boolean fetchLastLevelOnly) throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); List devices = new ArrayList<>(); List temperatures = new ArrayList<>(); @@ -331,6 +346,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { filter.setRootEntity(tenantId); filter.setDirection(EntitySearchDirection.FROM); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); + filter.setMaxLevel(maxLevel); + filter.setFetchLastLevelOnly(fetchLastLevelOnly); EntityDataSortOrder sortOrder = new EntityDataSortOrder( new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC @@ -383,7 +400,6 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { deviceService.deleteDevicesByTenantId(tenantId); } - @Test public void testHierarchicalFindDevicesWithAttributesByQuery() throws ExecutionException, InterruptedException { List assets = new ArrayList<>(); @@ -403,6 +419,8 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { filter.setRootEntity(tenantId); filter.setDirection(EntitySearchDirection.FROM); filter.setRelationType("Contains"); + filter.setMaxLevel(2); + filter.setFetchLastLevelOnly(true); EntityDataSortOrder sortOrder = new EntityDataSortOrder( new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC From 319d47eaaded3376bef1b8126bf9d9a148d432b8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 12 Oct 2021 18:48:28 +0300 Subject: [PATCH 25/34] fixed CE license headers --- .../dao/PostgreSqlDaoServiceTestSuite.java | 35 ++++++------------- .../server/dao/PostgreSqlInitializer.java | 35 ++++++------------- .../server/dao/service/DaoPostgreSqlTest.java | 35 ++++++------------- 3 files changed, 30 insertions(+), 75 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java index 915379fe74..b89c70e8f4 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.dao; diff --git a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java index 5f70b7ad44..429fa1d711 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java +++ b/dao/src/test/java/org/thingsboard/server/dao/PostgreSqlInitializer.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.dao; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java index 16da531369..43ed1f4d02 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java @@ -1,32 +1,17 @@ /** - * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL + * Copyright © 2016-2021 The Thingsboard Authors * - * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * NOTICE: All information contained herein is, and remains - * the property of ThingsBoard, Inc. and its suppliers, - * if any. The intellectual and technical concepts contained - * herein are proprietary to ThingsBoard, Inc. - * and its suppliers and may be covered by U.S. and Foreign Patents, - * patents in process, and are protected by trade secret or copyright law. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Dissemination of this information or reproduction of this material is strictly forbidden - * unless prior written permission is obtained from COMPANY. - * - * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, - * managers or contractors who have executed Confidentiality and Non-disclosure agreements - * explicitly covering such access. - * - * The copyright notice above does not evidence any actual or intended publication - * or disclosure of this source code, which includes - * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. - * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, - * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT - * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, - * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. - * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION - * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, - * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.thingsboard.server.dao.service; From 2d4ac40e25a2495ee4427ecf73ef4c3c9fe6040b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 13 Oct 2021 16:51:49 +0300 Subject: [PATCH 26/34] Refactor searchTenantIdRecursive for rule chains import --- .../org/thingsboard/server/dao/rule/BaseRuleChainService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index fa1c4ec173..487f39f2cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -481,7 +481,9 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } if (isTenantId) { ObjectNode objNode = (ObjectNode) node; - objNode.put("id", tenantId.getId().toString()); + if (objNode.has("id")) { + objNode.put("id", tenantId.getId().toString()); + } } else { for (JsonNode jsonNode : node) { searchTenantIdRecursive(tenantId, jsonNode); From b4f230dbc8dca59d43a831ab22d20d5a562475aa Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 13 Oct 2021 20:35:10 +0300 Subject: [PATCH 27/34] Improve markdown component. Add new helps. --- ui-ngx/package.json | 2 +- .../dynamic-component-factory.service.ts | 10 +- ui-ngx/src/app/core/services/help.service.ts | 29 ++- ...custom-action-pretty-editor.component.html | 3 +- ...ction-pretty-resources-tabs.component.html | 3 +- .../widget-action-dialog.component.html | 1 + .../widget/data-key-config.component.html | 2 + .../widget/lib/markdown-widget.component.html | 2 +- .../components/widget/widget.component.ts | 2 +- .../components/help-markdown.component.html | 2 +- .../components/help-markdown.component.scss | 6 +- .../components/help-markdown.component.ts | 14 +- .../components/help-popup.component.html | 32 +-- .../components/help-popup.component.scss | 46 ++++ .../shared/components/help-popup.component.ts | 38 ++- .../json-form/json-form.component.ts | 5 +- .../shared/components/markdown.component.html | 21 ++ .../shared/components/markdown.component.scss | 20 ++ .../shared/components/markdown.component.ts | 154 ++++++++++++ .../components/marked-options.service.ts | 4 +- .../shared/components/popover.component.ts | 164 +------------ .../app/shared/components/popover.service.ts | 190 +++++++++++++++ ui-ngx/src/app/shared/components/tokens.ts | 24 ++ ui-ngx/src/app/shared/models/constants.ts | 2 +- ui-ngx/src/app/shared/shared.module.ts | 9 +- .../en_US/widget/action/custom_action_fn.md | 92 ++++++++ .../widget/action/custom_additional_params.md | 46 ++++ .../widget/action/custom_pretty_action_fn.md | 74 ++++++ .../custom_pretty_create_dialog_html.md | 160 +++++++++++++ .../custom_pretty_create_dialog_js.md | 132 +++++++++++ .../custom_pretty_edit_dialog_html.md | 192 +++++++++++++++ .../examples/custom_pretty_edit_dialog_js.md | 220 ++++++++++++++++++ .../action/show_widget_action_cell_fn.md | 47 +++- .../action/show_widget_action_header_fn.md | 6 +- .../widget/config/datakey_generation_fn.md | 59 +++++ .../widget/config/datakey_postprocess_fn.md | 56 +++++ .../widget/lib/tooltip_value_format_fn.md | 35 ++- ui-ngx/src/styles.scss | 15 +- ui-ngx/yarn.lock | 52 ++--- 39 files changed, 1724 insertions(+), 247 deletions(-) create mode 100644 ui-ngx/src/app/shared/components/markdown.component.html create mode 100644 ui-ngx/src/app/shared/components/markdown.component.scss create mode 100644 ui-ngx/src/app/shared/components/markdown.component.ts create mode 100644 ui-ngx/src/app/shared/components/popover.service.ts create mode 100644 ui-ngx/src/app/shared/components/tokens.ts create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/custom_action_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/custom_additional_params.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/custom_pretty_action_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/examples/custom_pretty_create_dialog_html.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/examples/custom_pretty_create_dialog_js.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/examples/custom_pretty_edit_dialog_html.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/action/examples/custom_pretty_edit_dialog_js.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/config/datakey_generation_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/widget/config/datakey_postprocess_fn.md diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 20d03503be..2aaaff7b66 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -73,7 +73,7 @@ "ngx-drag-drop": "^2.0.0", "ngx-flowchart": "git://github.com/thingsboard/ngx-flowchart.git#master", "ngx-hm-carousel": "^2.0.0-rc.1", - "ngx-markdown": "^10.1.1", + "ngx-markdown": "^11.1.3", "ngx-sharebuttons": "^8.0.5", "ngx-translate-messageformat-compiler": "^4.9.0", "objectpath": "^2.0.0", diff --git a/ui-ngx/src/app/core/services/dynamic-component-factory.service.ts b/ui-ngx/src/app/core/services/dynamic-component-factory.service.ts index f840c89c26..fa8d63be62 100644 --- a/ui-ngx/src/app/core/services/dynamic-component-factory.service.ts +++ b/ui-ngx/src/app/core/services/dynamic-component-factory.service.ts @@ -55,11 +55,12 @@ export class DynamicComponentFactoryService { public createDynamicComponentFactory( componentType: Type, template: string, - modules?: Type[]): Observable> { + modules?: Type[], + preserveWhitespaces?: boolean): Observable> { const dymamicComponentFactorySubject = new ReplaySubject>(); import('@angular/compiler').then( () => { - const comp = this.createDynamicComponent(componentType, template); + const comp = this.createDynamicComponent(componentType, template, preserveWhitespaces); let moduleImports: Type[] = [CommonModule]; if (modules) { moduleImports = [...moduleImports, ...modules]; @@ -103,10 +104,11 @@ export class DynamicComponentFactoryService { } } - private createDynamicComponent(componentType: Type, template: string): Type { + private createDynamicComponent(componentType: Type, template: string, preserveWhitespaces?: boolean): Type { // noinspection AngularMissingOrInvalidDeclarationInModule return Component({ - template + template, + preserveWhitespaces })(componentType); } diff --git a/ui-ngx/src/app/core/services/help.service.ts b/ui-ngx/src/app/core/services/help.service.ts index 5494c415c4..2ed8863b4d 100644 --- a/ui-ngx/src/app/core/services/help.service.ts +++ b/ui-ngx/src/app/core/services/help.service.ts @@ -18,7 +18,8 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { TranslateService } from '@ngx-translate/core'; import { Observable, of } from 'rxjs'; -import { catchError, tap } from 'rxjs/operators'; +import { catchError, mergeMap, tap } from 'rxjs/operators'; +import { helpBaseUrl } from '@shared/models/constants'; const NOT_FOUND_CONTENT = '## Not found'; @@ -27,6 +28,8 @@ const NOT_FOUND_CONTENT = '## Not found'; }) export class HelpService { + private helpBaseUrl = helpBaseUrl; + private helpCache: {[lang: string]: {[key: string]: string}} = {}; constructor( @@ -52,6 +55,9 @@ export class HelpService { return of(NOT_FOUND_CONTENT); } }), + mergeMap((content) => { + return this.processIncludes(this.processVariables(content)); + }), tap((content) => { let langContent = this.helpCache[lang]; if (!langContent) { @@ -68,4 +74,25 @@ export class HelpService { return this.http.get(`/assets/help/${lang}/${key}.md`, {responseType: 'text'} ); } + private processVariables(content: string): string { + const baseUrlReg = /\${baseUrl}/g; + return content.replace(baseUrlReg, this.helpBaseUrl); + } + + private processIncludes(content: string): Observable { + const includesRule = /{% include (.*) %}/; + const match = includesRule.exec(content); + if (match) { + const key = match[1]; + return this.getHelpContent(key).pipe( + mergeMap((include) => { + content = content.replace(match[0], include); + return this.processIncludes(content); + }) + ); + } else { + return of(content); + } + } + } diff --git a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-editor.component.html b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-editor.component.html index 642f739f8e..7921f519f4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-editor.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-editor.component.html @@ -46,7 +46,8 @@ [functionArgs]="['$event', 'widgetContext', 'entityId', 'entityName', 'htmlTemplate', 'additionalParams', 'entityLabel']" [disableUndefinedCheck]="true" [validationArgs]="[]" - [editorCompleter]="customPrettyActionEditorCompleter"> + [editorCompleter]="customPrettyActionEditorCompleter" + helpId="widget/action/custom_pretty_action_fn"> diff --git a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.html b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.html index d5893c47ab..53351918e2 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/action/custom-action-pretty-resources-tabs.component.html @@ -96,7 +96,8 @@ [functionArgs]="['$event', 'widgetContext', 'entityId', 'entityName', 'htmlTemplate', 'additionalParams', 'entityLabel']" [disableUndefinedCheck]="true" [validationArgs]="[]" - [editorCompleter]="customPrettyActionEditorCompleter"> + [editorCompleter]="customPrettyActionEditorCompleter" + helpId="widget/action/custom_pretty_action_fn"> diff --git a/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html b/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html index a04b9a16e8..2cf9b427ec 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/action/widget-action-dialog.component.html @@ -224,6 +224,7 @@ [globalVariables]="functionScopeVariables" [validationArgs]="[]" [editorCompleter]="customActionEditorCompleter" + helpId="widget/action/custom_action_fn" > diff --git a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html index 5987520d7b..457bbdd777 100644 --- a/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/data-key-config.component.html @@ -70,6 +70,7 @@ [globalVariables]="functionScopeVariables" [validationArgs]="[[1, 1],[1, '1']]" resultType="any" + helpId="widget/config/datakey_generation_fn" formControlName="funcBody"> @@ -82,6 +83,7 @@ [globalVariables]="functionScopeVariables" [validationArgs]="[[1, 1, 1, 1, 1],[1, '1', '1', 1, '1']]" resultType="any" + helpId="widget/config/datakey_postprocess_fn" formControlName="postFuncBody">