From 653a1225aa92c50c3ef850bc6ccd7ddab71c9af3 Mon Sep 17 00:00:00 2001 From: desoliture Date: Wed, 12 Jan 2022 14:57:45 +0200 Subject: [PATCH 001/178] refactor services to use tenantService.getTenantById instead of tenantDao.getById --- .../server/dao/alarm/BaseAlarmService.java | 6 +++--- .../server/dao/asset/BaseAssetService.java | 7 ++++--- .../server/dao/customer/CustomerServiceImpl.java | 9 +++------ .../server/dao/dashboard/DashboardServiceImpl.java | 7 ++++--- .../server/dao/device/DeviceProfileServiceImpl.java | 6 +++--- .../server/dao/device/DeviceServiceImpl.java | 6 +++--- .../server/dao/edge/EdgeServiceImpl.java | 6 +++--- .../server/dao/entity/AbstractEntityService.java | 1 + .../dao/entityview/EntityViewServiceImpl.java | 6 +++--- .../server/dao/ota/BaseOtaPackageService.java | 7 ++++--- .../server/dao/resource/BaseResourceService.java | 11 ++++++----- .../server/dao/rule/BaseRuleChainService.java | 7 ++++--- .../server/dao/tenant/TenantServiceImpl.java | 9 +++++---- .../dao/usagerecord/ApiUsageStateServiceImpl.java | 13 +++++++------ .../server/dao/user/UserServiceImpl.java | 11 ++++++----- .../server/dao/widget/WidgetTypeServiceImpl.java | 7 ++++--- .../server/dao/widget/WidgetsBundleServiceImpl.java | 7 ++++--- 17 files changed, 67 insertions(+), 59 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index eb6d5c778e..e2e1d54935 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -81,7 +81,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ private AlarmDao alarmDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private EntityService entityService; @@ -430,7 +430,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ if (alarm.getTenantId() == null) { throw new DataValidationException("Alarm should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(alarm.getTenantId(), alarm.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(alarm.getTenantId()); if (tenant == null) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index d88494c6e1..d9f337841b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -55,7 +55,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.ArrayList; import java.util.Arrays; @@ -86,7 +86,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ private AssetDao assetDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -416,7 +416,8 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ if (asset.getTenantId() == null) { throw new DataValidationException("Asset should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, asset.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(asset.getTenantId()); + // FIXME: 12.01.22 if (tenant == null) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 89690e5a3b..3b2376f98d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -42,7 +42,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; @@ -66,7 +66,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom private UserService userService; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private AssetService assetService; @@ -74,9 +74,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom @Autowired private DeviceService deviceService; - @Autowired - private EntityViewService entityViewService; - @Autowired private DashboardService dashboardService; @@ -213,7 +210,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom if (customer.getTenantId() == null) { throw new DataValidationException("Customer should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, customer.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(customer.getTenantId()); if (tenant == null) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index d460fef60b..c2a6813eda 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -45,7 +45,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -62,7 +62,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb private DashboardInfoDao dashboardInfoDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -308,7 +308,8 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb if (dashboard.getTenantId() == null) { throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, dashboard.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(dashboard.getTenantId()); + // FIXME: 12.01.22 if (tenant == null) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 18a8a1ecde..254b84889b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -81,7 +81,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.QueueService; import java.util.Arrays; @@ -127,7 +127,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D private DeviceService deviceService; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CacheManager cacheManager; @@ -375,7 +375,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D if (deviceProfile.getTenantId() == null) { throw new DataValidationException("Device profile should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(deviceProfile.getTenantId(), deviceProfile.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(deviceProfile.getTenantId()); if (tenant == null) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f70d1c1ffe..fbc7f5a23a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -81,7 +81,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import javax.annotation.Nullable; import java.util.ArrayList; @@ -117,7 +117,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe private DeviceDao deviceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -741,7 +741,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe if (device.getTenantId() == null) { throw new DataValidationException("Device should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(device.getTenantId(), device.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(device.getTenantId()); if (tenant == null) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index e77e24fb93..281848a68d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -60,7 +60,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserService; import javax.annotation.Nullable; @@ -96,7 +96,7 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic private EdgeDao edgeDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -413,7 +413,7 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic if (edge.getTenantId() == null) { throw new DataValidationException("Edge should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(edge.getTenantId(), edge.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(edge.getTenantId()); if (tenant == null) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 970712fc19..a71c1e8ec6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -40,6 +40,7 @@ public abstract class AbstractEntityService { public static final String INCORRECT_EDGE_ID = "Incorrect edgeId "; public static final String INCORRECT_PAGE_LINK = "Incorrect page link "; + @Lazy @Autowired protected RelationService relationService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 33ea3d635f..f2aa6423fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -52,7 +52,7 @@ import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import javax.annotation.Nullable; import java.util.ArrayList; @@ -87,7 +87,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti private EntityViewDao entityViewDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -432,7 +432,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti if (entityView.getTenantId() == null) { throw new DataValidationException("Entity view should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, entityView.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(entityView.getTenantId()); if (tenant == null) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 205b2c447b..734c96b1c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -46,7 +46,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.nio.ByteBuffer; import java.util.Collections; @@ -66,7 +66,7 @@ public class BaseOtaPackageService implements OtaPackageService { public static final String INCORRECT_OTA_PACKAGE_ID = "Incorrect otaPackageId "; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - private final TenantDao tenantDao; + private final TenantService tenantService; private final DeviceProfileDao deviceProfileDao; private final OtaPackageDao otaPackageDao; private final OtaPackageInfoDao otaPackageInfoDao; @@ -357,7 +357,8 @@ public class BaseOtaPackageService implements OtaPackageService { if (otaPackageInfo.getTenantId() == null) { throw new DataValidationException("OtaPackage should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(otaPackageInfo.getTenantId(), otaPackageInfo.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(otaPackageInfo.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("OtaPackage is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index f442f5acea..d8f8976dcb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.List; import java.util.Optional; @@ -52,13 +52,13 @@ public class BaseResourceService implements ResourceService { public static final String INCORRECT_RESOURCE_ID = "Incorrect resourceId "; private final TbResourceDao resourceDao; private final TbResourceInfoDao resourceInfoDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final TbTenantProfileCache tenantProfileCache; - public BaseResourceService(TbResourceDao resourceDao, TbResourceInfoDao resourceInfoDao, TenantDao tenantDao, @Lazy TbTenantProfileCache tenantProfileCache) { + public BaseResourceService(TbResourceDao resourceDao, TbResourceInfoDao resourceInfoDao, TenantService tenantService, @Lazy TbTenantProfileCache tenantProfileCache) { this.resourceDao = resourceDao; this.resourceInfoDao = resourceInfoDao; - this.tenantDao = tenantDao; + this.tenantService = tenantService; this.tenantProfileCache = tenantProfileCache; } @@ -183,7 +183,8 @@ public class BaseResourceService implements ResourceService { resource.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, resource.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(resource.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 6b990a7c6c..f649dc9a0a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -59,7 +59,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.ArrayList; import java.util.Collection; @@ -94,7 +94,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private RuleNodeDao ruleNodeDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -726,7 +726,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getTenantId() == null || ruleChain.getTenantId().isNullUid()) { throw new DataValidationException("Rule chain should be assigned to tenant!"); } - Tenant tenant = tenantDao.findById(tenantId, ruleChain.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(ruleChain.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 7314f54b39..6f5189b4ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -21,6 +21,8 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; @@ -34,7 +36,6 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.resource.ResourceService; @@ -66,6 +67,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private TenantProfileService tenantProfileService; @Autowired + @Lazy private UserService userService; @Autowired @@ -83,9 +85,6 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private ApiUsageStateService apiUsageStateService; - @Autowired - private EntityViewService entityViewService; - @Autowired private WidgetsBundleService widgetsBundleService; @@ -126,6 +125,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Override + @Transactional public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -143,6 +143,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Override + @Transactional public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 880f507b44..d9cedb5b77 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfigurat import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.tenant.TenantProfileDao; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -52,11 +52,11 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final ApiUsageStateDao apiUsageStateDao; private final TenantProfileDao tenantProfileDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final TimeseriesService tsService; - public ApiUsageStateServiceImpl(TenantDao tenantDao, ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, TimeseriesService tsService) { - this.tenantDao = tenantDao; + public ApiUsageStateServiceImpl(TenantService tenantService, ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, TimeseriesService tsService) { + this.tenantService = tenantService; this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; this.tsService = tsService; @@ -114,7 +114,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (entityId.getEntityType() == EntityType.TENANT && !entityId.equals(TenantId.SYS_TENANT_ID)) { tenantId = (TenantId) entityId; - Tenant tenant = tenantDao.findById(tenantId, tenantId.getId()); + Tenant tenant = tenantService.findTenantById(tenantId); TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenant.getTenantProfileId().getId()); TenantProfileConfiguration configuration = tenantProfile.getProfileData().getConfiguration(); @@ -164,7 +164,8 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (apiUsageState.getTenantId() == null) { throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(requestTenantId, apiUsageState.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(apiUsageState.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 1ed5811ffa..171cb5fbd1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.HashMap; import java.util.Map; @@ -80,20 +80,20 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic private final UserDao userDao; private final UserCredentialsDao userCredentialsDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final CustomerDao customerDao; private final TbTenantProfileCache tenantProfileCache; private final ApplicationEventPublisher eventPublisher; public UserServiceImpl(UserDao userDao, UserCredentialsDao userCredentialsDao, - TenantDao tenantDao, + @Lazy TenantService tenantService, CustomerDao customerDao, @Lazy TbTenantProfileCache tenantProfileCache, ApplicationEventPublisher eventPublisher) { this.userDao = userDao; this.userCredentialsDao = userCredentialsDao; - this.tenantDao = tenantDao; + this.tenantService = tenantService; this.customerDao = customerDao; this.tenantProfileCache = tenantProfileCache; this.eventPublisher = eventPublisher; @@ -448,7 +448,8 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic + " already present in database!"); } if (!tenantId.getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, user.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(user.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("User is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index d8a8ea3e35..6952f4a7d2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -30,7 +30,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.List; @@ -44,7 +44,7 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { private WidgetTypeDao widgetTypeDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private WidgetsBundleDao widgetsBundleService; @@ -138,7 +138,8 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { widgetTypeDetails.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetTypeDetails.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(widgetTypeDetails.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 4f7bd93eca..559481df74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -31,7 +31,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.ArrayList; import java.util.List; @@ -48,7 +48,7 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { private WidgetsBundleDao widgetsBundleDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private WidgetTypeService widgetTypeService; @@ -162,7 +162,8 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { widgetsBundle.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetsBundle.getTenantId().getId()); + Tenant tenant = tenantService.findTenantById(widgetsBundle.getTenantId()); + // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } From 2a93edbac48e398dfc4fd95a8e6a23f49fbf1552 Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 13 Jan 2022 12:35:01 +0200 Subject: [PATCH 002/178] add cache support for TenantService --- application/src/main/resources/thingsboard.yml | 3 +++ .../org/thingsboard/server/common/data/CacheConstants.java | 1 + .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a502afb297..6ff108968d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -407,6 +407,9 @@ caffeine: tenantProfiles: timeToLiveInMinutes: "${CACHE_SPECS_TENANT_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_TENANT_PROFILES_MAX_SIZE:10000}" + tenants: + timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" + maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" deviceProfiles: timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_DEVICE_PROFILES_MAX_SIZE:10000}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index ced7a64a0f..19c2a76561 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -26,6 +26,7 @@ public class CacheConstants { public static final String CLAIM_DEVICES_CACHE = "claimDevices"; public static final String SECURITY_SETTINGS_CACHE = "securitySettings"; public static final String TENANT_PROFILE_CACHE = "tenantProfiles"; + public static final String TENANTS_CACHE = "tenants"; public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 6f5189b4ca..47179049a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -20,6 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -48,6 +50,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import static org.thingsboard.server.common.data.CacheConstants.TENANTS_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -104,6 +107,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private RpcService rpcService; @Override + @Cacheable(cacheNames = TENANTS_CACHE, key = "#tenantId", condition = "#tenantId!=null") public Tenant findTenantById(TenantId tenantId) { log.trace("Executing findTenantById [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -126,6 +130,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override @Transactional + @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenant.id", condition = "#tenant.id!=null") public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -144,6 +149,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override @Transactional + @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenantId", condition = "#tenantId!=null") public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); From cc84069c683798e85a9d40bf4ea23ffe998cae6f Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 13 Jan 2022 13:42:13 +0200 Subject: [PATCH 003/178] add corresponding tests for caching in TenantService --- .../dao/service/BaseTenantServiceTest.java | 83 ++++++++++++++++++- .../resources/application-test.properties | 3 + 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 29b0558c71..066857f500 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -18,6 +18,12 @@ package org.thingsboard.server.dao.service; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; @@ -27,16 +33,26 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.tenant.TenantDao; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; +import java.util.Objects; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; public abstract class BaseTenantServiceTest extends AbstractServiceTest { private IdComparator idComparator = new IdComparator<>(); + @SpyBean + protected TenantDao tenantDao; + + @Autowired + CacheManager cacheManager; + @Test public void testSaveTenant() { Tenant tenant = new Tenant(); @@ -275,4 +291,69 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTenantProfileId(isolatedTenantProfile.getId()); tenantService.saveTenant(tenant); } + + @Test + public void testGettingTenantAddItToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Tenant cache manager is null").evict(savedTenant.getId()); + + Mockito.verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + Cache.ValueWrapper cachedTenant = + Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedTenant); + + for (int i = 0; i < 100; i++) { + tenantService.findTenantById(savedTenant.getId()); + } + Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdatingExistingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Cache.ValueWrapper cachedTenant = + Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + + savedTenant.setTitle("My new tenant"); + savedTenant = tenantService.saveTenant(savedTenant); + + Mockito.reset(tenantDao); + + cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); + + Mockito.verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testRemovingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Cache.ValueWrapper cachedTenant = + Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + + tenantService.deleteTenant(savedTenant.getId()); + cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); + } } diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index a257c8fbce..4165523701 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -34,6 +34,9 @@ caffeine.specs.claimDevices.maxSize=100000 caffeine.specs.tenantProfiles.timeToLiveInMinutes=1440 caffeine.specs.tenantProfiles.maxSize=100000 +caffeine.specs.tenants.timeToLiveInMinutes=1440 +caffeine.specs.tenants.maxSize=100000 + caffeine.specs.deviceProfiles.timeToLiveInMinutes=1440 caffeine.specs.deviceProfiles.maxSize=100000 From ba030b07b16805372025433159f871e5b153f821 Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 13 Jan 2022 14:12:59 +0200 Subject: [PATCH 004/178] add todo notes --- .../java/org/thingsboard/server/dao/alarm/BaseAlarmService.java | 1 + .../java/org/thingsboard/server/dao/asset/BaseAssetService.java | 2 +- .../thingsboard/server/dao/customer/CustomerServiceImpl.java | 1 + .../thingsboard/server/dao/dashboard/DashboardServiceImpl.java | 2 +- .../thingsboard/server/dao/device/DeviceProfileServiceImpl.java | 1 + .../org/thingsboard/server/dao/device/DeviceServiceImpl.java | 1 + .../java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java | 1 + .../server/dao/entityview/EntityViewServiceImpl.java | 1 + 8 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index e2e1d54935..e6df02c854 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -431,6 +431,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ throw new DataValidationException("Alarm should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(alarm.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index d9f337841b..94477635af 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -417,7 +417,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ throw new DataValidationException("Asset should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(asset.getTenantId()); - // FIXME: 12.01.22 + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 3b2376f98d..f8726e9d16 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -211,6 +211,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom throw new DataValidationException("Customer should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(customer.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index c2a6813eda..aaf5e5d584 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -309,7 +309,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(dashboard.getTenantId()); - // FIXME: 12.01.22 + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 254b84889b..b71822907d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -376,6 +376,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D throw new DataValidationException("Device profile should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(deviceProfile.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index fbc7f5a23a..0f1f830a4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -742,6 +742,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe throw new DataValidationException("Device should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(device.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 281848a68d..7e9bd64741 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -414,6 +414,7 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic throw new DataValidationException("Edge should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(edge.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index f2aa6423fa..7c3d16dd1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -433,6 +433,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti throw new DataValidationException("Entity view should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(entityView.getTenantId()); + // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } From 279389bbdfff2180a2b5be8b20fda8b663b33b1f Mon Sep 17 00:00:00 2001 From: desoliture Date: Fri, 14 Jan 2022 16:09:02 +0200 Subject: [PATCH 005/178] refactor async methods used in tenant deletion transaction tenant deleting should be transactional, but some services use async methods, what corrupting transaction execution. Withal most of the refactored methods are using instant .get() after getting future, and it is the same if we use non-async methods. Add non-async methods in interfaces of services and dao and use it for tenant deletion process --- .../dao/entityview/EntityViewService.java | 2 ++ .../server/dao/asset/BaseAssetService.java | 11 +++------- .../server/dao/device/DeviceServiceImpl.java | 22 +++++-------------- .../dao/entity/AbstractEntityService.java | 2 +- .../server/dao/entityview/EntityViewDao.java | 1 + .../dao/entityview/EntityViewServiceImpl.java | 21 ++++++++++++++++++ .../dao/sql/entityview/JpaEntityViewDao.java | 8 +++++++ 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index 832a7822ee..abb79e48fa 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -72,6 +72,8 @@ public interface EntityViewService { ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId); + List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + void deleteEntityView(TenantId tenantId, EntityViewId entityViewId); void deleteEntityViewsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 94477635af..3e178919f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -168,14 +168,9 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ deleteEntityRelations(tenantId, assetId); Asset asset = assetDao.findById(tenantId, assetId.getId()); - try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(asset.getTenantId(), assetId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete asset that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for assetId [{}]", assetId, e); - throw new RuntimeException("Exception while finding entity views for assetId [" + assetId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(asset.getTenantId(), assetId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete asset that has entity views!"); } removeAssetFromCacheByName(asset.getTenantId(), asset.getName()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 0f1f830a4d..778ee3e789 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -344,14 +344,9 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe Device device = deviceDao.findById(tenantId, deviceId.getId()); final String deviceName = device.getName(); - try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete device that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", deviceId, e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + deviceId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), deviceId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete device that has entity views!"); } DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, deviceId); @@ -568,14 +563,9 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe public Device assignDeviceToTenant(TenantId tenantId, Device device) { log.trace("Executing assignDeviceToTenant [{}][{}]", tenantId, device); - try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); - if (!CollectionUtils.isEmpty(entityViews)) { - throw new DataValidationException("Can't assign device that has entity views to another tenant!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + device.getId() + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), device.getId()); + if (!CollectionUtils.isEmpty(entityViews)) { + throw new DataValidationException("Can't assign device that has entity views to another tenant!"); } eventService.removeEvents(device.getTenantId(), device.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index a71c1e8ec6..acb49bb730 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -83,7 +83,7 @@ public abstract class AbstractEntityService { protected void checkAssignedEntityViewsToEdge(TenantId tenantId, EntityId entityId, EdgeId edgeId) { try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId).get(); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(tenantId, entityId); if (entityViews != null && !entityViews.isEmpty()) { EntityView entityView = entityViews.get(0); // TODO: @voba - refactor this blocking operation diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index c4a9e0d871..b7c65e7c8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -179,4 +179,5 @@ public interface EntityViewDao extends Dao { String type, PageLink pageLink); + List findEntityViewsByTenantIdAndEntityId(UUID tenantId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 7c3d16dd1a..30b32e4caf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -302,6 +302,27 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti } } + @Override + public List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findEntityViewsByTenantIdAndEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(entityId.getId(), "Incorrect entityId" + entityId); + + List tenantIdAndEntityId = new ArrayList<>(); + tenantIdAndEntityId.add(tenantId); + tenantIdAndEntityId.add(entityId); + + Cache cache = cacheManager.getCache(ENTITY_VIEW_CACHE); + List fromCache = cache.get(tenantIdAndEntityId, List.class); + if (fromCache != null) { + return fromCache; + } else { + List result = entityViewDao.findEntityViewsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + cache.putIfAbsent(tenantIdAndEntityId, result); + return result; + } + } + @CacheEvict(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityViewId}") @Override public void deleteEntityView(TenantId tenantId, EntityViewId entityViewId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 35f81a860c..676caed2ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -200,4 +200,12 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao findEntityViewsByTenantIdAndEntityId(UUID tenantId, UUID entityId) { + return DaoUtil.convertDataList( + entityViewRepository.findAllByTenantIdAndEntityId( + tenantId, entityId) + ); + } } From 3d4503945594427291240a2889a81b02ccf251ae Mon Sep 17 00:00:00 2001 From: desoliture Date: Fri, 14 Jan 2022 18:14:49 +0200 Subject: [PATCH 006/178] add test for tenant deletion method add corresponding test to make sure all related entities are also deleted when tenant is deleted --- .../dao/service/AbstractServiceTest.java | 4 + .../dao/service/BaseTenantServiceTest.java | 289 +++++++++++++++++- 2 files changed, 292 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 2d4fd72179..fd497b7c50 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -59,6 +59,7 @@ import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; +import org.thingsboard.server.dao.rpc.RpcService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -163,6 +164,9 @@ public abstract class AbstractServiceTest { @Autowired protected OtaPackageService otaPackageService; + @Autowired + protected RpcService rpcService; + public class IdComparator implements Comparator { @Override public int compare(D o1, D o2) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 066857f500..1a716fdac3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -23,28 +23,70 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rpc.RpcStatus; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.tenant.TenantDao; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Set; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public abstract class BaseTenantServiceTest extends AbstractServiceTest { - + + public static final String TITLE = "My firmware"; + private static final String FILE_NAME = "filename.txt"; + private static final String VERSION = "v1.0"; + private static final String CONTENT_TYPE = "text/plain"; + private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256; + private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; + private static final long DATA_SIZE = 1L; + private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{(int) DATA_SIZE}); + private static final String URL = "http://firmware.test.org"; + + private IdComparator idComparator = new IdComparator<>(); @SpyBean @@ -356,4 +398,249 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); } + + @Test + public void testDeleteTenantDeletingAllRelatedEntities() throws Exception { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test tenant profile"); + TenantProfile savedProfile = tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(savedProfile.getId()); + Tenant savedTenant = tenantService.saveTenant(tenant); + + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setEmail("tenantAdmin@test.com"); + user.setFirstName("tenantAdmin"); + user.setLastName("tenantAdmin"); + user.setTenantId(savedTenant.getId()); + User savedUser = userService.saveUser(user); + + Customer customer = new Customer(); + customer.setTitle("Test customer"); + customer.setTenantId(savedTenant.getId()); + customer.setEmail("testCustomer@test.com"); + Customer savedCustomer = customerService.saveCustomer(customer); + + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setTenantId(savedTenant.getId()); + widgetsBundle.setTitle("Test widgets bundle"); + widgetsBundle.setAlias("TestWidgetsBundle"); + widgetsBundle.setDescription("Just a simple widgets bundle"); + WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); + + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(savedTenant.getId()); + deviceProfile.setTransportType(DeviceTransportType.MQTT); + deviceProfile.setName("Test device profile"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setTransportConfiguration(new MqttDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + + Device device = new Device(); + device.setCustomerId(savedCustomer.getId()); + device.setTenantId(savedTenant.getId()); + device.setType("Test type"); + device.setName("TestType"); + device.setLabel("Test type"); + device.setDeviceProfileId(savedDeviceProfile.getId()); + Device savedDevice = deviceService.saveDevice(device); + + EntityView entityView = new EntityView(); + entityView.setEntityId(savedDevice.getId()); + entityView.setTenantId(savedTenant.getId()); + entityView.setCustomerId(savedCustomer.getId()); + entityView.setType("Test type"); + entityView.setName("Test entity view"); + entityView.setStartTimeMs(0); + entityView.setEndTimeMs(840000); + EntityView savedEntityView = entityViewService.saveEntityView(entityView); + + Asset asset = new Asset(); + asset.setTenantId(savedTenant.getId()); + asset.setCustomerId(savedCustomer.getId()); + asset.setType("Test asset type"); + asset.setName("Test asset type"); + asset.setLabel("Test asset type"); + Asset savedAsset = assetService.saveAsset(asset); + + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setTitle("Test dashboard"); + dashboard.setAssignedCustomers(Set.of(savedCustomer.toShortCustomerInfo())); + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(savedTenant.getId()); + ruleChain.setName("Test rule chain"); + ruleChain.setType(RuleChainType.CORE); + RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); + + Edge edge = constructEdge(savedTenant.getId(), "Test edge", "Simple"); + Edge savedEdge = edgeService.saveEdge(edge, false); + + OtaPackage otaPackage = createFirmware(savedTenant.getId(), "1", savedDeviceProfile.getId()); + OtaPackage savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + + + TbResource resource = new TbResource(); + resource.setTenantId(savedTenant.getId()); + resource.setTitle("Test resource"); + resource.setResourceType(ResourceType.LWM2M_MODEL); + resource.setFileName(FILE_NAME); + resource.setResourceKey("Test resource key"); + resource.setData("Some super test data"); + TbResource savedResource = resourceService.saveResource(resource); + + ApiUsageState defaultApiUsageState = apiUsageStateService + .createDefaultApiUsageState(savedTenant.getId(), savedCustomer.getId()); + + Rpc rpc = new Rpc(); + rpc.setTenantId(savedTenant.getId()); + rpc.setDeviceId(savedDevice.getId()); + rpc.setStatus(RpcStatus.QUEUED); + rpc.setRequest(JacksonUtil.toJsonNode("{}")); + Rpc savedRpc = rpcService.save(rpc); + + + tenantService.deleteTenant(savedTenant.getId()); + + Assert.assertNull(tenantService.findTenantById(savedTenant.getId())); + Assert.assertNull(tenantService.findTenantById(savedTenant.getId())); + Assert.assertNull(customerService.findCustomerById(savedTenant.getId(), savedCustomer.getId())); + + + PageLink pageLinkCustomer = new PageLink(1000); + PageData pageDataCustomer = customerService + .findCustomersByTenantId(savedTenant.getId(), pageLinkCustomer); + Assert.assertFalse(pageDataCustomer.hasNext()); + Assert.assertEquals(0, pageDataCustomer.getTotalElements()); + + + Assert.assertNull( + widgetsBundleService.findWidgetsBundleById(savedTenant.getId(), savedWidgetsBundle.getId()) + ); + List widgetsBundlesByTenantId = + widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(savedTenant.getId()); + Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); + + + Assert.assertNull(entityViewService.findEntityViewById( + savedTenant.getId(), savedEntityView.getId() + )); + List entityViews = + entityViewService.findEntityViewsByTenantIdAndEntityId( + savedTenant.getId(), savedDevice.getId()); + Assert.assertTrue(entityViews.isEmpty()); + + + Assert.assertNull(assetService.findAssetById( + savedTenant.getId(), savedAsset.getId() + )); + PageLink pageLinkAssets = new PageLink(1000); + PageData assets = + assetService.findAssetsByTenantId(savedTenant.getId(), pageLinkAssets); + Assert.assertFalse(assets.hasNext()); + Assert.assertEquals(0, assets.getTotalElements()); + + + Assert.assertNull(deviceService.findDeviceById( + savedTenant.getId(), savedDevice.getId() + )); + PageLink pageLinkDevices = new PageLink(1000); + PageData devices = + deviceService.findDevicesByTenantId(savedTenant.getId(), pageLinkDevices); + Assert.assertFalse(devices.hasNext()); + Assert.assertEquals(0, devices.getTotalElements()); + + + Assert.assertNull(deviceProfileService.findDeviceProfileById( + savedTenant.getId(), savedDeviceProfile.getId() + )); + PageLink pageLinkDeviceProfiles = new PageLink(1000); + PageData profiles = + deviceProfileService.findDeviceProfiles(savedTenant.getId(), pageLinkDeviceProfiles); + Assert.assertFalse(profiles.hasNext()); + Assert.assertEquals(0, profiles.getTotalElements()); + + + Assert.assertNull(dashboardService.findDashboardById( + savedTenant.getId(), savedDashboard.getId() + )); + PageLink pageLinkDashboards = new PageLink(1000); + PageData dashboards = + dashboardService.findDashboardsByTenantId(savedTenant.getId(), pageLinkDashboards); + Assert.assertFalse(dashboards.hasNext()); + Assert.assertEquals(0, dashboards.getTotalElements()); + + + Assert.assertNull(edgeService.findEdgeById(savedTenant.getId(), savedEdge.getId())); + PageLink pageLinkEdges = new PageLink(1000); + PageData edges = edgeService.findEdgesByTenantId(savedTenant.getId(), pageLinkEdges); + Assert.assertFalse(edges.hasNext()); + Assert.assertEquals(0, edges.getTotalElements()); + + + PageLink pageLinkTenantAdmins = new PageLink(1000); + PageData tenantAdmins = + userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); + Assert.assertFalse(tenantAdmins.hasNext()); + Assert.assertEquals(0, tenantAdmins.getTotalElements()); + + + Assert.assertNull(userService.findUserById(savedTenant.getId(), savedUser.getId())); + PageLink pageLinkUsers = new PageLink(1000); + PageData users = + userService.findUsersByTenantId(savedTenant.getId(), pageLinkUsers); + Assert.assertFalse(users.hasNext()); + Assert.assertEquals(0, users.getTotalElements()); + + + Assert.assertNull(ruleChainService.findRuleChainById(savedTenant.getId(), savedRuleChain.getId())); + Assert.assertNull(apiUsageStateService.findTenantApiUsageState(savedTenant.getId())); + + + Assert.assertNull(resourceService.findResourceById(savedTenant.getId(), savedResource.getId())); + PageLink pageLinkResources = new PageLink(1000); + PageData tenantResources = + resourceService.findAllTenantResourcesByTenantId(savedTenant.getId(), pageLinkResources); + Assert.assertFalse(tenantResources.hasNext()); + Assert.assertEquals(0, tenantResources.getTotalElements()); + + + Assert.assertNull( + otaPackageService.findOtaPackageById( + savedTenant.getId(), savedOtaPackage.getId() + ) + ); + PageLink pageLinkOta = new PageLink(1000); + PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(savedTenant.getId(), pageLinkOta); + Assert.assertFalse(pageDataOta.hasNext()); + Assert.assertEquals(0, pageDataOta.getTotalElements()); + + + Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); + } + + private OtaPackage createFirmware(TenantId tenantId, String version, DeviceProfileId deviceProfileId) { + OtaPackage firmware = new OtaPackage(); + firmware.setTenantId(tenantId); + firmware.setDeviceProfileId(deviceProfileId); + firmware.setType(FIRMWARE); + firmware.setTitle(TITLE); + firmware.setVersion(version); + firmware.setFileName(FILE_NAME); + firmware.setContentType(CONTENT_TYPE); + firmware.setChecksumAlgorithm(CHECKSUM_ALGORITHM); + firmware.setChecksum(CHECKSUM); + firmware.setData(DATA); + firmware.setDataSize(DATA_SIZE); + return otaPackageService.saveOtaPackage(firmware); + } } From a0a658c3a4e44fc7a37127549eaef50e462d27a6 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 17 Jan 2022 12:19:05 +0200 Subject: [PATCH 007/178] refactor test for Tenant deletion --- .../dao/service/BaseTenantServiceTest.java | 456 ++++++++++-------- 1 file changed, 251 insertions(+), 205 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 1a716fdac3..7644ee86e7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -24,7 +24,6 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -76,17 +75,6 @@ import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public abstract class BaseTenantServiceTest extends AbstractServiceTest { - public static final String TITLE = "My firmware"; - private static final String FILE_NAME = "filename.txt"; - private static final String VERSION = "v1.0"; - private static final String CONTENT_TYPE = "text/plain"; - private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256; - private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; - private static final long DATA_SIZE = 1L; - private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{(int) DATA_SIZE}); - private static final String URL = "http://firmware.test.org"; - - private IdComparator idComparator = new IdComparator<>(); @SpyBean @@ -401,155 +389,111 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { @Test public void testDeleteTenantDeletingAllRelatedEntities() throws Exception { - TenantProfile tenantProfile = new TenantProfile(); - tenantProfile.setName("Test tenant profile"); - TenantProfile savedProfile = tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); - - Tenant tenant = new Tenant(); - tenant.setTitle("My tenant"); - tenant.setTenantProfileId(savedProfile.getId()); - Tenant savedTenant = tenantService.saveTenant(tenant); - - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setEmail("tenantAdmin@test.com"); - user.setFirstName("tenantAdmin"); - user.setLastName("tenantAdmin"); - user.setTenantId(savedTenant.getId()); - User savedUser = userService.saveUser(user); - - Customer customer = new Customer(); - customer.setTitle("Test customer"); - customer.setTenantId(savedTenant.getId()); - customer.setEmail("testCustomer@test.com"); - Customer savedCustomer = customerService.saveCustomer(customer); - - WidgetsBundle widgetsBundle = new WidgetsBundle(); - widgetsBundle.setTenantId(savedTenant.getId()); - widgetsBundle.setTitle("Test widgets bundle"); - widgetsBundle.setAlias("TestWidgetsBundle"); - widgetsBundle.setDescription("Just a simple widgets bundle"); - WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); - - DeviceProfile deviceProfile = new DeviceProfile(); - deviceProfile.setTenantId(savedTenant.getId()); - deviceProfile.setTransportType(DeviceTransportType.MQTT); - deviceProfile.setName("Test device profile"); - deviceProfile.setType(DeviceProfileType.DEFAULT); - - DeviceProfileData profileData = new DeviceProfileData(); - profileData.setTransportConfiguration(new MqttDeviceProfileTransportConfiguration()); - deviceProfile.setProfileData(profileData); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - - Device device = new Device(); - device.setCustomerId(savedCustomer.getId()); - device.setTenantId(savedTenant.getId()); - device.setType("Test type"); - device.setName("TestType"); - device.setLabel("Test type"); - device.setDeviceProfileId(savedDeviceProfile.getId()); - Device savedDevice = deviceService.saveDevice(device); - - EntityView entityView = new EntityView(); - entityView.setEntityId(savedDevice.getId()); - entityView.setTenantId(savedTenant.getId()); - entityView.setCustomerId(savedCustomer.getId()); - entityView.setType("Test type"); - entityView.setName("Test entity view"); - entityView.setStartTimeMs(0); - entityView.setEndTimeMs(840000); - EntityView savedEntityView = entityViewService.saveEntityView(entityView); - - Asset asset = new Asset(); - asset.setTenantId(savedTenant.getId()); - asset.setCustomerId(savedCustomer.getId()); - asset.setType("Test asset type"); - asset.setName("Test asset type"); - asset.setLabel("Test asset type"); - Asset savedAsset = assetService.saveAsset(asset); - - Dashboard dashboard = new Dashboard(); - dashboard.setTenantId(savedTenant.getId()); - dashboard.setTitle("Test dashboard"); - dashboard.setAssignedCustomers(Set.of(savedCustomer.toShortCustomerInfo())); - Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); - - RuleChain ruleChain = new RuleChain(); - ruleChain.setTenantId(savedTenant.getId()); - ruleChain.setName("Test rule chain"); - ruleChain.setType(RuleChainType.CORE); - RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); - - Edge edge = constructEdge(savedTenant.getId(), "Test edge", "Simple"); - Edge savedEdge = edgeService.saveEdge(edge, false); - - OtaPackage otaPackage = createFirmware(savedTenant.getId(), "1", savedDeviceProfile.getId()); - OtaPackage savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); - - - TbResource resource = new TbResource(); - resource.setTenantId(savedTenant.getId()); - resource.setTitle("Test resource"); - resource.setResourceType(ResourceType.LWM2M_MODEL); - resource.setFileName(FILE_NAME); - resource.setResourceKey("Test resource key"); - resource.setData("Some super test data"); - TbResource savedResource = resourceService.saveResource(resource); - - ApiUsageState defaultApiUsageState = apiUsageStateService - .createDefaultApiUsageState(savedTenant.getId(), savedCustomer.getId()); - - Rpc rpc = new Rpc(); - rpc.setTenantId(savedTenant.getId()); - rpc.setDeviceId(savedDevice.getId()); - rpc.setStatus(RpcStatus.QUEUED); - rpc.setRequest(JacksonUtil.toJsonNode("{}")); - Rpc savedRpc = rpcService.save(rpc); - + TenantProfile savedProfile = createAndSaveTenantProfile(); + Tenant savedTenant = createAndSaveTenant(savedProfile); + User savedUser = createAndSaveUserFor(savedTenant); + Customer savedCustomer = createAndSaveCustomerFor(savedTenant); + WidgetsBundle savedWidgetsBundle = createAndSaveWidgetBundleFor(savedTenant); + DeviceProfile savedDeviceProfile = createAndSaveDeviceProfileWithProfileDataFor(savedTenant); + Device savedDevice = createAndSaveDeviceFor(savedTenant, savedCustomer, savedDeviceProfile); + EntityView savedEntityView = createAndSaveEntityViewFor(savedTenant, savedCustomer, savedDevice); + Asset savedAsset = createAndSaveAssetFor(savedTenant, savedCustomer); + Dashboard savedDashboard = createAndSaveDashboardFor(savedTenant, savedCustomer); + RuleChain savedRuleChain = createAndSaveRuleChainFor(savedTenant); + Edge savedEdge = createAndSaveEdgeFor(savedTenant); + OtaPackage savedOtaPackage = createAndSaveOtaPackageFor(savedTenant, savedDeviceProfile); + TbResource savedResource = createAndSaveResourceFor(savedTenant); + Rpc savedRpc = createAndSaveRpcFor(savedTenant, savedDevice); tenantService.deleteTenant(savedTenant.getId()); Assert.assertNull(tenantService.findTenantById(savedTenant.getId())); - Assert.assertNull(tenantService.findTenantById(savedTenant.getId())); - Assert.assertNull(customerService.findCustomerById(savedTenant.getId(), savedCustomer.getId())); - - - PageLink pageLinkCustomer = new PageLink(1000); - PageData pageDataCustomer = customerService - .findCustomersByTenantId(savedTenant.getId(), pageLinkCustomer); - Assert.assertFalse(pageDataCustomer.hasNext()); - Assert.assertEquals(0, pageDataCustomer.getTotalElements()); - + assertCustomerIsDeleted(savedTenant, savedCustomer); + assertWidgetsBundleIsDeleted(savedTenant, savedWidgetsBundle); + assertEntityViewIsDeleted(savedTenant, savedDevice, savedEntityView); + assertAssetIsDeleted(savedTenant, savedAsset); + assertDeviceIsDeleted(savedTenant, savedDevice); + assertDeviceProfileIsDeleted(savedTenant, savedDeviceProfile); + assertDashboardIsDeleted(savedTenant, savedDashboard); + assertEdgeIsDeletd(savedTenant, savedEdge); + assertTenantAdminIsDeleted(savedTenant); + assertUserIsDeleted(savedTenant, savedUser); + Assert.assertNull(ruleChainService.findRuleChainById(savedTenant.getId(), savedRuleChain.getId())); + Assert.assertNull(apiUsageStateService.findTenantApiUsageState(savedTenant.getId())); + assertResourceIsDeleted(savedTenant, savedResource); + assertOtaPAckageIsDeleted(savedTenant, savedOtaPackage); + Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); + } + private void assertOtaPAckageIsDeleted(Tenant savedTenant, OtaPackage savedOtaPackage) { Assert.assertNull( - widgetsBundleService.findWidgetsBundleById(savedTenant.getId(), savedWidgetsBundle.getId()) + otaPackageService.findOtaPackageById( + savedTenant.getId(), savedOtaPackage.getId() + ) ); - List widgetsBundlesByTenantId = - widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(savedTenant.getId()); - Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); + PageLink pageLinkOta = new PageLink(1000); + PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(savedTenant.getId(), pageLinkOta); + Assert.assertFalse(pageDataOta.hasNext()); + Assert.assertEquals(0, pageDataOta.getTotalElements()); + } + private void assertResourceIsDeleted(Tenant savedTenant, TbResource savedResource) { + Assert.assertNull(resourceService.findResourceById(savedTenant.getId(), savedResource.getId())); + PageLink pageLinkResources = new PageLink(1000); + PageData tenantResources = + resourceService.findAllTenantResourcesByTenantId(savedTenant.getId(), pageLinkResources); + Assert.assertFalse(tenantResources.hasNext()); + Assert.assertEquals(0, tenantResources.getTotalElements()); + } - Assert.assertNull(entityViewService.findEntityViewById( - savedTenant.getId(), savedEntityView.getId() - )); - List entityViews = - entityViewService.findEntityViewsByTenantIdAndEntityId( - savedTenant.getId(), savedDevice.getId()); - Assert.assertTrue(entityViews.isEmpty()); + private void assertUserIsDeleted(Tenant savedTenant, User savedUser) { + Assert.assertNull(userService.findUserById(savedTenant.getId(), savedUser.getId())); + PageLink pageLinkUsers = new PageLink(1000); + PageData users = + userService.findUsersByTenantId(savedTenant.getId(), pageLinkUsers); + Assert.assertFalse(users.hasNext()); + Assert.assertEquals(0, users.getTotalElements()); + } + private void assertTenantAdminIsDeleted(Tenant savedTenant) { + PageLink pageLinkTenantAdmins = new PageLink(1000); + PageData tenantAdmins = + userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); + Assert.assertFalse(tenantAdmins.hasNext()); + Assert.assertEquals(0, tenantAdmins.getTotalElements()); + } - Assert.assertNull(assetService.findAssetById( - savedTenant.getId(), savedAsset.getId() + private void assertEdgeIsDeletd(Tenant savedTenant, Edge savedEdge) { + Assert.assertNull(edgeService.findEdgeById(savedTenant.getId(), savedEdge.getId())); + PageLink pageLinkEdges = new PageLink(1000); + PageData edges = edgeService.findEdgesByTenantId(savedTenant.getId(), pageLinkEdges); + Assert.assertFalse(edges.hasNext()); + Assert.assertEquals(0, edges.getTotalElements()); + } + + private void assertDashboardIsDeleted(Tenant savedTenant, Dashboard savedDashboard) { + Assert.assertNull(dashboardService.findDashboardById( + savedTenant.getId(), savedDashboard.getId() )); - PageLink pageLinkAssets = new PageLink(1000); - PageData assets = - assetService.findAssetsByTenantId(savedTenant.getId(), pageLinkAssets); - Assert.assertFalse(assets.hasNext()); - Assert.assertEquals(0, assets.getTotalElements()); + PageLink pageLinkDashboards = new PageLink(1000); + PageData dashboards = + dashboardService.findDashboardsByTenantId(savedTenant.getId(), pageLinkDashboards); + Assert.assertFalse(dashboards.hasNext()); + Assert.assertEquals(0, dashboards.getTotalElements()); + } + private void assertDeviceProfileIsDeleted(Tenant savedTenant, DeviceProfile savedDeviceProfile) { + Assert.assertNull(deviceProfileService.findDeviceProfileById( + savedTenant.getId(), savedDeviceProfile.getId() + )); + PageLink pageLinkDeviceProfiles = new PageLink(1000); + PageData profiles = + deviceProfileService.findDeviceProfiles(savedTenant.getId(), pageLinkDeviceProfiles); + Assert.assertFalse(profiles.hasNext()); + Assert.assertEquals(0, profiles.getTotalElements()); + } + private void assertDeviceIsDeleted(Tenant savedTenant, Device savedDevice) { Assert.assertNull(deviceService.findDeviceById( savedTenant.getId(), savedDevice.getId() )); @@ -558,89 +502,191 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { deviceService.findDevicesByTenantId(savedTenant.getId(), pageLinkDevices); Assert.assertFalse(devices.hasNext()); Assert.assertEquals(0, devices.getTotalElements()); + } + private void assertAssetIsDeleted(Tenant savedTenant, Asset savedAsset) { + Assert.assertNull(assetService.findAssetById( + savedTenant.getId(), savedAsset.getId() + )); + PageLink pageLinkAssets = new PageLink(1000); + PageData assets = + assetService.findAssetsByTenantId(savedTenant.getId(), pageLinkAssets); + Assert.assertFalse(assets.hasNext()); + Assert.assertEquals(0, assets.getTotalElements()); + } - Assert.assertNull(deviceProfileService.findDeviceProfileById( - savedTenant.getId(), savedDeviceProfile.getId() + private void assertEntityViewIsDeleted(Tenant savedTenant, Device savedDevice, EntityView savedEntityView) { + Assert.assertNull(entityViewService.findEntityViewById( + savedTenant.getId(), savedEntityView.getId() )); - PageLink pageLinkDeviceProfiles = new PageLink(1000); - PageData profiles = - deviceProfileService.findDeviceProfiles(savedTenant.getId(), pageLinkDeviceProfiles); - Assert.assertFalse(profiles.hasNext()); - Assert.assertEquals(0, profiles.getTotalElements()); + List entityViews = + entityViewService.findEntityViewsByTenantIdAndEntityId( + savedTenant.getId(), savedDevice.getId()); + Assert.assertTrue(entityViews.isEmpty()); + } + private void assertWidgetsBundleIsDeleted(Tenant savedTenant, WidgetsBundle savedWidgetsBundle) { + Assert.assertNull( + widgetsBundleService.findWidgetsBundleById(savedTenant.getId(), savedWidgetsBundle.getId()) + ); + List widgetsBundlesByTenantId = + widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(savedTenant.getId()); + Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); + } - Assert.assertNull(dashboardService.findDashboardById( - savedTenant.getId(), savedDashboard.getId() - )); - PageLink pageLinkDashboards = new PageLink(1000); - PageData dashboards = - dashboardService.findDashboardsByTenantId(savedTenant.getId(), pageLinkDashboards); - Assert.assertFalse(dashboards.hasNext()); - Assert.assertEquals(0, dashboards.getTotalElements()); + private void assertCustomerIsDeleted(Tenant savedTenant, Customer savedCustomer) { + Assert.assertNull(customerService.findCustomerById(savedTenant.getId(), savedCustomer.getId())); + PageLink pageLinkCustomer = new PageLink(1000); + PageData pageDataCustomer = customerService + .findCustomersByTenantId(savedTenant.getId(), pageLinkCustomer); + Assert.assertFalse(pageDataCustomer.hasNext()); + Assert.assertEquals(0, pageDataCustomer.getTotalElements()); + } + private Rpc createAndSaveRpcFor(Tenant savedTenant, Device savedDevice) { + Rpc rpc = new Rpc(); + rpc.setTenantId(savedTenant.getId()); + rpc.setDeviceId(savedDevice.getId()); + rpc.setStatus(RpcStatus.QUEUED); + rpc.setRequest(JacksonUtil.toJsonNode("{}")); + return rpcService.save(rpc); + } - Assert.assertNull(edgeService.findEdgeById(savedTenant.getId(), savedEdge.getId())); - PageLink pageLinkEdges = new PageLink(1000); - PageData edges = edgeService.findEdgesByTenantId(savedTenant.getId(), pageLinkEdges); - Assert.assertFalse(edges.hasNext()); - Assert.assertEquals(0, edges.getTotalElements()); + private TbResource createAndSaveResourceFor(Tenant savedTenant) { + TbResource resource = new TbResource(); + resource.setTenantId(savedTenant.getId()); + resource.setTitle("Test resource"); + resource.setResourceType(ResourceType.LWM2M_MODEL); + resource.setFileName("filename.txt"); + resource.setResourceKey("Test resource key"); + resource.setData("Some super test data"); + return resourceService.saveResource(resource); + } + private OtaPackage createAndSaveOtaPackageFor(Tenant savedTenant, DeviceProfile savedDeviceProfile) { + OtaPackage otaPackage = createFirmware(savedTenant.getId(), savedDeviceProfile.getId()); + return otaPackageService.saveOtaPackage(otaPackage); + } - PageLink pageLinkTenantAdmins = new PageLink(1000); - PageData tenantAdmins = - userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); - Assert.assertFalse(tenantAdmins.hasNext()); - Assert.assertEquals(0, tenantAdmins.getTotalElements()); + private Edge createAndSaveEdgeFor(Tenant savedTenant) { + Edge edge = constructEdge(savedTenant.getId(), "Test edge", "Simple"); + return edgeService.saveEdge(edge, false); + } + private RuleChain createAndSaveRuleChainFor(Tenant savedTenant) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(savedTenant.getId()); + ruleChain.setName("Test rule chain"); + ruleChain.setType(RuleChainType.CORE); + return ruleChainService.saveRuleChain(ruleChain); + } - Assert.assertNull(userService.findUserById(savedTenant.getId(), savedUser.getId())); - PageLink pageLinkUsers = new PageLink(1000); - PageData users = - userService.findUsersByTenantId(savedTenant.getId(), pageLinkUsers); - Assert.assertFalse(users.hasNext()); - Assert.assertEquals(0, users.getTotalElements()); + private Dashboard createAndSaveDashboardFor(Tenant savedTenant, Customer savedCustomer) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setTitle("Test dashboard"); + dashboard.setAssignedCustomers(Set.of(savedCustomer.toShortCustomerInfo())); + return dashboardService.saveDashboard(dashboard); + } + private Asset createAndSaveAssetFor(Tenant savedTenant, Customer savedCustomer) { + Asset asset = new Asset(); + asset.setTenantId(savedTenant.getId()); + asset.setCustomerId(savedCustomer.getId()); + asset.setType("Test asset type"); + asset.setName("Test asset type"); + asset.setLabel("Test asset type"); + return assetService.saveAsset(asset); + } - Assert.assertNull(ruleChainService.findRuleChainById(savedTenant.getId(), savedRuleChain.getId())); - Assert.assertNull(apiUsageStateService.findTenantApiUsageState(savedTenant.getId())); + private EntityView createAndSaveEntityViewFor(Tenant savedTenant, Customer savedCustomer, Device savedDevice) { + EntityView entityView = new EntityView(); + entityView.setEntityId(savedDevice.getId()); + entityView.setTenantId(savedTenant.getId()); + entityView.setCustomerId(savedCustomer.getId()); + entityView.setType("Test type"); + entityView.setName("Test entity view"); + entityView.setStartTimeMs(0); + entityView.setEndTimeMs(840000); + return entityViewService.saveEntityView(entityView); + } + private Device createAndSaveDeviceFor(Tenant savedTenant, Customer savedCustomer, DeviceProfile savedDeviceProfile) { + Device device = new Device(); + device.setCustomerId(savedCustomer.getId()); + device.setTenantId(savedTenant.getId()); + device.setType("Test type"); + device.setName("TestType"); + device.setLabel("Test type"); + device.setDeviceProfileId(savedDeviceProfile.getId()); + return deviceService.saveDevice(device); + } - Assert.assertNull(resourceService.findResourceById(savedTenant.getId(), savedResource.getId())); - PageLink pageLinkResources = new PageLink(1000); - PageData tenantResources = - resourceService.findAllTenantResourcesByTenantId(savedTenant.getId(), pageLinkResources); - Assert.assertFalse(tenantResources.hasNext()); - Assert.assertEquals(0, tenantResources.getTotalElements()); + private DeviceProfile createAndSaveDeviceProfileWithProfileDataFor(Tenant savedTenant) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(savedTenant.getId()); + deviceProfile.setTransportType(DeviceTransportType.MQTT); + deviceProfile.setName("Test device profile"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setTransportConfiguration(new MqttDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + private WidgetsBundle createAndSaveWidgetBundleFor(Tenant savedTenant) { + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setTenantId(savedTenant.getId()); + widgetsBundle.setTitle("Test widgets bundle"); + widgetsBundle.setAlias("TestWidgetsBundle"); + widgetsBundle.setDescription("Just a simple widgets bundle"); + return widgetsBundleService.saveWidgetsBundle(widgetsBundle); + } - Assert.assertNull( - otaPackageService.findOtaPackageById( - savedTenant.getId(), savedOtaPackage.getId() - ) - ); - PageLink pageLinkOta = new PageLink(1000); - PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(savedTenant.getId(), pageLinkOta); - Assert.assertFalse(pageDataOta.hasNext()); - Assert.assertEquals(0, pageDataOta.getTotalElements()); + private Customer createAndSaveCustomerFor(Tenant savedTenant) { + Customer customer = new Customer(); + customer.setTitle("Test customer"); + customer.setTenantId(savedTenant.getId()); + customer.setEmail("testCustomer@test.com"); + return customerService.saveCustomer(customer); + } + private User createAndSaveUserFor(Tenant savedTenant) { + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setEmail("tenantAdmin@test.com"); + user.setFirstName("tenantAdmin"); + user.setLastName("tenantAdmin"); + user.setTenantId(savedTenant.getId()); + return userService.saveUser(user); + } - Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); + private Tenant createAndSaveTenant(TenantProfile savedProfile) { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(savedProfile.getId()); + return tenantService.saveTenant(tenant); + } + + private TenantProfile createAndSaveTenantProfile() { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test tenant profile"); + return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); } - private OtaPackage createFirmware(TenantId tenantId, String version, DeviceProfileId deviceProfileId) { + private OtaPackage createFirmware(TenantId tenantId, DeviceProfileId deviceProfileId) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId); firmware.setDeviceProfileId(deviceProfileId); firmware.setType(FIRMWARE); - firmware.setTitle(TITLE); - firmware.setVersion(version); - firmware.setFileName(FILE_NAME); - firmware.setContentType(CONTENT_TYPE); - firmware.setChecksumAlgorithm(CHECKSUM_ALGORITHM); - firmware.setChecksum(CHECKSUM); - firmware.setData(DATA); - firmware.setDataSize(DATA_SIZE); + firmware.setTitle("My firmware"); + firmware.setVersion("1"); + firmware.setFileName("filename.txt"); + firmware.setContentType("text/plain"); + firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + firmware.setData(ByteBuffer.wrap(new byte[]{(int) 1L})); + firmware.setDataSize(1L); return otaPackageService.saveOtaPackage(firmware); } } From 801e747b02f88a2e95ff3ff51d4784f1a26b2ee6 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 17 Jan 2022 12:52:04 +0200 Subject: [PATCH 008/178] fix names in test for Tenant deletion --- .../server/dao/service/BaseTenantServiceTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 7644ee86e7..06aaaff43d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -415,17 +415,17 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { assertDeviceIsDeleted(savedTenant, savedDevice); assertDeviceProfileIsDeleted(savedTenant, savedDeviceProfile); assertDashboardIsDeleted(savedTenant, savedDashboard); - assertEdgeIsDeletd(savedTenant, savedEdge); + assertEdgeIsDeleted(savedTenant, savedEdge); assertTenantAdminIsDeleted(savedTenant); assertUserIsDeleted(savedTenant, savedUser); Assert.assertNull(ruleChainService.findRuleChainById(savedTenant.getId(), savedRuleChain.getId())); Assert.assertNull(apiUsageStateService.findTenantApiUsageState(savedTenant.getId())); assertResourceIsDeleted(savedTenant, savedResource); - assertOtaPAckageIsDeleted(savedTenant, savedOtaPackage); + assertOtaPackageIsDeleted(savedTenant, savedOtaPackage); Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); } - private void assertOtaPAckageIsDeleted(Tenant savedTenant, OtaPackage savedOtaPackage) { + private void assertOtaPackageIsDeleted(Tenant savedTenant, OtaPackage savedOtaPackage) { Assert.assertNull( otaPackageService.findOtaPackageById( savedTenant.getId(), savedOtaPackage.getId() @@ -463,7 +463,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Assert.assertEquals(0, tenantAdmins.getTotalElements()); } - private void assertEdgeIsDeletd(Tenant savedTenant, Edge savedEdge) { + private void assertEdgeIsDeleted(Tenant savedTenant, Edge savedEdge) { Assert.assertNull(edgeService.findEdgeById(savedTenant.getId(), savedEdge.getId())); PageLink pageLinkEdges = new PageLink(1000); PageData edges = edgeService.findEdgesByTenantId(savedTenant.getId(), pageLinkEdges); From bbf02b5f39ad5ed035a24884f211201a564fbeb8 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 17 Jan 2022 13:21:23 +0200 Subject: [PATCH 009/178] add timeout for tenant deletion transaction and fix test for tenant deletion --- .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 6 +++--- .../server/dao/service/BaseTenantServiceTest.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 47179049a1..d9cfd8647d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -107,7 +107,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private RpcService rpcService; @Override - @Cacheable(cacheNames = TENANTS_CACHE, key = "#tenantId", condition = "#tenantId!=null") + @Cacheable(cacheNames = TENANTS_CACHE, key = "#tenantId") public Tenant findTenantById(TenantId tenantId) { log.trace("Executing findTenantById [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -148,8 +148,8 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Override - @Transactional - @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenantId", condition = "#tenantId!=null") + @Transactional(timeout = 60 * 60) + @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenantId") public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 06aaaff43d..21bbddde9b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -423,6 +423,8 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { assertResourceIsDeleted(savedTenant, savedResource); assertOtaPackageIsDeleted(savedTenant, savedOtaPackage); Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); + + tenantProfileService.deleteTenantProfile(TenantId.SYS_TENANT_ID, savedProfile.getId()); } private void assertOtaPackageIsDeleted(Tenant savedTenant, OtaPackage savedOtaPackage) { From d9dbd273641ffa17aff9e50c6298d8b8162368f3 Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 17 Jan 2022 13:34:28 +0200 Subject: [PATCH 010/178] refactor test for tenant deletion --- .../dao/service/BaseTenantServiceTest.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 21bbddde9b..9ad900fb1a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -71,6 +71,8 @@ import java.util.Set; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public abstract class BaseTenantServiceTest extends AbstractServiceTest { @@ -328,12 +330,12 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTitle("My tenant"); Tenant savedTenant = tenantService.saveTenant(tenant); - Mockito.reset(tenantDao); + reset(tenantDao); Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Tenant cache manager is null").evict(savedTenant.getId()); - Mockito.verify(tenantDao, Mockito.times(0)).findById(any(), any()); + verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); - Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); Cache.ValueWrapper cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); @@ -342,7 +344,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { for (int i = 0; i < 100; i++) { tenantService.findTenantById(savedTenant.getId()); } - Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); tenantService.deleteTenant(savedTenant.getId()); } @@ -360,14 +362,14 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { savedTenant.setTitle("My new tenant"); savedTenant = tenantService.saveTenant(savedTenant); - Mockito.reset(tenantDao); + reset(tenantDao); cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); - Mockito.verify(tenantDao, Mockito.times(0)).findById(any(), any()); + verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); - Mockito.verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); tenantService.deleteTenant(savedTenant.getId()); } From a39b56c93cbe0e6c69eec1cfa6070fc0498c5b0c Mon Sep 17 00:00:00 2001 From: desoliture Date: Mon, 17 Jan 2022 13:35:28 +0200 Subject: [PATCH 011/178] replace remained tenantDao usages to tenantService --- .../server/service/ttl/AlarmsCleanUpService.java | 6 +++--- .../server/service/ttl/rpc/RpcCleanUpService.java | 6 +++--- .../org/thingsboard/server/dao/tenant/TenantService.java | 2 ++ .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 7 +++++++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 1dfb4a67bf..5182b6ffe5 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -32,7 +32,7 @@ import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; @@ -50,7 +50,7 @@ public class AlarmsCleanUpService { @Value("${sql.ttl.alarms.removal_batch_size}") private Integer removalBatchSize; - private final TenantDao tenantDao; + private final TenantService tenantService; private final AlarmDao alarmDao; private final AlarmService alarmService; private final RelationService relationService; @@ -64,7 +64,7 @@ public class AlarmsCleanUpService { PageLink removalBatchRequest = new PageLink(removalBatchSize, 0 ); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index c0985eb4c1..3252d3df1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -43,7 +43,7 @@ public class RpcCleanUpService { @Value("${sql.ttl.rpc.enabled}") private boolean ttlTaskExecutionEnabled; - private final TenantDao tenantDao; + private final TenantService tenantService; private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; private final RpcDao rpcDao; @@ -54,7 +54,7 @@ public class RpcCleanUpService { PageLink tenantsBatchRequest = new PageLink(10_000, 0); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java index 95236ad8a3..01c4d9b4e2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java @@ -39,4 +39,6 @@ public interface TenantService { PageData findTenantInfos(PageLink pageLink); void deleteTenants(); + + PageData findTenantsIds(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index d9cfd8647d..ecca39b93c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -191,6 +191,13 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe tenantsRemover.removeEntities(new TenantId(EntityId.NULL_UUID), DEFAULT_TENANT_REGION); } + @Override + public PageData findTenantsIds(PageLink pageLink) { + log.trace("Executing deleteTenants"); + Validator.validatePageLink(pageLink); + return tenantDao.findTenantsIds(pageLink); + } + private DataValidator tenantValidator = new DataValidator() { @Override From 83ae5ba26284beaf97f69644ae365c03d0186d89 Mon Sep 17 00:00:00 2001 From: desoliture Date: Wed, 19 Jan 2022 16:37:51 +0200 Subject: [PATCH 012/178] refactor --- .../org/thingsboard/server/dao/tenant/TenantServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index ecca39b93c..50fe23f22b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -123,7 +123,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public ListenableFuture findTenantByIdAsync(TenantId callerId, TenantId tenantId) { - log.trace("Executing TenantIdAsync [{}]", tenantId); + log.trace("Executing findTenantByIdAsync [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); return tenantDao.findByIdAsync(callerId, tenantId.getId()); } @@ -193,7 +193,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public PageData findTenantsIds(PageLink pageLink) { - log.trace("Executing deleteTenants"); + log.trace("Executing findTenantsIds"); Validator.validatePageLink(pageLink); return tenantDao.findTenantsIds(pageLink); } From 16753d2f624a13b768758f7e2736c63591a06e85 Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 20 Jan 2022 16:50:40 +0200 Subject: [PATCH 013/178] refactor relation service and dao(*), remove redundant todo's, refactor tests (*): resolve immediate .get() invocation, which producing blocking (checkRelation method), by adding sync checkRelation method and refactor the usages of checkRelationAsync --- .../rpc/sync/DefaultEdgeRequestsService.java | 2 +- .../server/dao/relation/RelationService.java | 4 +- .../server/dao/alarm/BaseAlarmService.java | 1 - .../server/dao/asset/BaseAssetService.java | 1 - .../dao/customer/CustomerServiceImpl.java | 1 - .../dao/dashboard/DashboardServiceImpl.java | 1 - .../dao/device/DeviceProfileServiceImpl.java | 1 - .../server/dao/device/DeviceServiceImpl.java | 1 - .../server/dao/edge/EdgeServiceImpl.java | 1 - .../dao/entity/AbstractEntityService.java | 22 +- .../dao/entityview/EntityViewServiceImpl.java | 17 +- .../dao/relation/BaseRelationService.java | 12 +- .../server/dao/relation/RelationDao.java | 4 +- .../dao/resource/BaseResourceService.java | 1 - .../server/dao/rule/BaseRuleChainService.java | 1 - .../sql/entityview/EntityViewRepository.java | 1 - .../dao/sql/relation/JpaRelationDao.java | 8 +- .../usagerecord/ApiUsageStateServiceImpl.java | 1 - .../dao/widget/WidgetTypeServiceImpl.java | 1 - .../dao/widget/WidgetsBundleServiceImpl.java | 1 - .../service/BaseOtaPackageServiceTest.java | 36 +- .../dao/service/BaseRelationServiceTest.java | 16 +- .../dao/service/BaseTenantServiceTest.java | 320 ++++++++---------- .../engine/action/TbCreateRelationNode.java | 2 +- .../engine/action/TbDeleteRelationNode.java | 2 +- .../engine/filter/TbCheckRelationNode.java | 2 +- .../action/TbCreateRelationNodeTest.java | 6 +- 27 files changed, 218 insertions(+), 248 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index a1f8fc89ec..80320a94c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -348,7 +348,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { if (entityViews != null && !entityViews.isEmpty()) { List> futures = new ArrayList<>(); for (EntityView entityView : entityViews) { - ListenableFuture future = relationService.checkRelation(tenantId, edge.getId(), entityView.getId(), + ListenableFuture future = relationService.checkRelationAsync(tenantId, edge.getId(), entityView.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE); futures.add(future); Futures.addCallback(future, new FutureCallback<>() { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 87ac26b41e..364eb2653b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -34,7 +34,9 @@ import java.util.List; */ public interface RelationService { - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + Boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index e6df02c854..e2e1d54935 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -431,7 +431,6 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ throw new DataValidationException("Alarm should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(alarm.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 3e178919f6..2610aff3e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -412,7 +412,6 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ throw new DataValidationException("Asset should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(asset.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index f8726e9d16..3b2376f98d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -211,7 +211,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom throw new DataValidationException("Customer should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(customer.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index aaf5e5d584..a27d69f7ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -309,7 +309,6 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(dashboard.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index b71822907d..254b84889b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -376,7 +376,6 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D throw new DataValidationException("Device profile should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(deviceProfile.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 778ee3e789..00d393c4bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -732,7 +732,6 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe throw new DataValidationException("Device should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(device.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 7e9bd64741..281848a68d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -414,7 +414,6 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic throw new DataValidationException("Edge should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(edge.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index acb49bb730..759ff937a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -82,20 +82,16 @@ public abstract class AbstractEntityService { } protected void checkAssignedEntityViewsToEdge(TenantId tenantId, EntityId entityId, EdgeId edgeId) { - try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(tenantId, entityId); - if (entityViews != null && !entityViews.isEmpty()) { - EntityView entityView = entityViews.get(0); - // TODO: @voba - refactor this blocking operation - Boolean relationExists = relationService.checkRelation(tenantId, edgeId, entityView.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE).get(); - if (relationExists) { - throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); - } + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(tenantId, entityId); + if (entityViews != null && !entityViews.isEmpty()) { + EntityView entityView = entityViews.get(0); + Boolean relationExists = relationService.checkRelation( + tenantId, edgeId, entityView.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE + ); + if (relationExists) { + throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); } - } catch (Exception e) { - log.error("[{}] Exception while finding entity views for entityId [{}]", tenantId, entityId, e); - throw new RuntimeException("Exception while finding entity views for entityId [" + entityId + "]", e); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 30b32e4caf..17bccd93c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -308,9 +308,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti validateId(tenantId, INCORRECT_TENANT_ID + tenantId); validateId(entityId.getId(), "Incorrect entityId" + entityId); - List tenantIdAndEntityId = new ArrayList<>(); - tenantIdAndEntityId.add(tenantId); - tenantIdAndEntityId.add(entityId); + List tenantIdAndEntityId = List.of(tenantId, entityId); Cache cache = cacheManager.getCache(ENTITY_VIEW_CACHE); List fromCache = cache.get(tenantIdAndEntityId, List.class); @@ -366,15 +364,10 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti throw new DataValidationException("Can't assign entityView to edge from different tenant!"); } - try { - Boolean relationExists = relationService.checkRelation(tenantId, edgeId, entityView.getEntityId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE).get(); - if (!relationExists) { - throw new DataValidationException("Can't assign entity view to edge because related device/asset doesn't assigned to edge!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception during relation check", e); - throw new RuntimeException("Exception during relation check", e); + Boolean relationExists = relationService.checkRelation(tenantId, edgeId, entityView.getEntityId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE); + if (!relationExists) { + throw new DataValidationException("Can't assign entity view to edge because related device/asset doesn't assigned to edge!"); } try { diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 996c0801ff..1f5f4ffbe2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -30,11 +30,8 @@ import org.springframework.cache.annotation.Caching; import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; @@ -77,10 +74,15 @@ public class BaseRelationService implements RelationService { private CacheManager cacheManager; @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing checkRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); - return relationDao.checkRelation(tenantId, from, to, relationType, typeGroup); + return relationDao.checkRelationAsync(tenantId, from, to, relationType, typeGroup); + } + + @Override + public Boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + return null; } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #to, #relationType, #typeGroup}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 85d333b04a..5df4f0420a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -41,7 +41,9 @@ public interface RelationDao { ListenableFuture> findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + Boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index d8f8976dcb..2511a7ec6a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -184,7 +184,6 @@ public class BaseResourceService implements ResourceService { } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { Tenant tenant = tenantService.findTenantById(resource.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index f649dc9a0a..1ef88f5450 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -727,7 +727,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC throw new DataValidationException("Rule chain should be assigned to tenant!"); } Tenant tenant = tenantService.findTenantById(ruleChain.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 45c5886d7c..05989f3a83 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,7 +20,6 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; -import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index fbedef6c4e..268abd58e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -101,11 +101,17 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); return service.submit(() -> relationRepository.existsById(key)); } + @Override + public Boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); + return relationRepository.existsById(key); + } + @Override public ListenableFuture getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index d9cedb5b77..8cb4e17598 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -165,7 +165,6 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(apiUsageState.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 6952f4a7d2..e2fe57ca74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -139,7 +139,6 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { Tenant tenant = tenantService.findTenantById(widgetTypeDetails.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 559481df74..2bea7f3ade 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -163,7 +163,6 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { Tenant tenant = tenantService.findTenantById(widgetsBundle.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 8fcadcd812..2e937ed164 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -97,26 +97,26 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "1"); + createAndSaveFirmware(tenantId, "1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); thrown.expect(DataValidationException.class); thrown.expectMessage(String.format("Failed to create the ota package, files size limit is exhausted %d bytes!", DATA_SIZE)); - createFirmware(tenantId, "2"); + createAndSaveFirmware(tenantId, "2"); } @Test public void sumDataSizeByTenantId() { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "0.1"); + createAndSaveFirmware(tenantId, "0.1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; List packages = new ArrayList<>(maxSumDataSize); for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createFirmware(tenantId, "0." + i)); + packages.add(createAndSaveFirmware(tenantId, "0." + i)); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -419,15 +419,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testSaveFirmwareWithExistingTitleAndVersion() { - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage with such title and version already exists!"); - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); } @Test public void testDeleteFirmwareWithReferenceByDevice() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); Device device = new Device(); device.setTenantId(tenantId); @@ -448,7 +448,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testUpdateDeviceProfileId() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); try { thrown.expect(DataValidationException.class); @@ -494,7 +494,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testFindFirmwareById() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -520,7 +520,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testDeleteFirmware() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -533,7 +533,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - OtaPackageInfo info = new OtaPackageInfo(createFirmware(tenantId, VERSION + i)); + OtaPackageInfo info = new OtaPackageInfo(createAndSaveFirmware(tenantId, VERSION + i)); info.setHasData(true); firmwares.add(info); } @@ -580,7 +580,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantIdAndHasData() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createFirmware(tenantId, VERSION + i)))); + firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i)))); } OtaPackageInfo firmwareWithUrl = new OtaPackageInfo(); @@ -696,7 +696,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } - private OtaPackage createFirmware(TenantId tenantId, String version) { + private OtaPackage createAndSaveFirmware(TenantId tenantId, String version) { + return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId)); + } + + public static OtaPackage createFirmware( + TenantId tenantId, + String version, + DeviceProfileId deviceProfileId + ) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId); firmware.setDeviceProfileId(deviceProfileId); @@ -709,6 +717,6 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setChecksum(CHECKSUM); firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - return otaPackageService.saveOtaPackage(firmware); + return firmware; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java index 27dc7f812a..4e7dd0a784 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java @@ -57,13 +57,13 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(saveRelation(relation)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); } @Test @@ -80,9 +80,9 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, relationA).get()); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); } @@ -118,9 +118,9 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertNull(relationService.deleteEntityRelationsAsync(SYSTEM_TENANT_ID, childId).get()); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 9ad900fb1a..b36467a307 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -46,9 +47,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rpc.Rpc; @@ -62,7 +61,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.tenant.TenantDao; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -71,9 +69,8 @@ import java.util.Set; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; +import static org.assertj.core.api.Assertions.assertThat; public abstract class BaseTenantServiceTest extends AbstractServiceTest { @@ -85,6 +82,13 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { @Autowired CacheManager cacheManager; + private Cache tenantCache; + + @Before + public void setup() { + tenantCache = cacheManager.getCache(CacheConstants.TENANTS_CACHE); + } + @Test public void testSaveTenant() { Tenant tenant = new Tenant(); @@ -330,15 +334,15 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTitle("My tenant"); Tenant savedTenant = tenantService.saveTenant(tenant); - reset(tenantDao); - Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Tenant cache manager is null").evict(savedTenant.getId()); + Mockito.reset(tenantDao); + Objects.requireNonNull(tenantCache, "Tenant cache manager is null").evict(savedTenant.getId()); verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedTenant); for (int i = 0; i < 100; i++) { @@ -356,15 +360,15 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Tenant savedTenant = tenantService.saveTenant(tenant); Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); savedTenant.setTitle("My new tenant"); savedTenant = tenantService.saveTenant(savedTenant); - reset(tenantDao); + Mockito.reset(tenantDao); - cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); verify(tenantDao, Mockito.times(0)).findById(any(), any()); @@ -381,184 +385,168 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Tenant savedTenant = tenantService.saveTenant(tenant); Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); tenantService.deleteTenant(savedTenant.getId()); - cachedTenant = Objects.requireNonNull(cacheManager.getCache(CacheConstants.TENANTS_CACHE), "Cache manager is null!").get(savedTenant.getId()); + cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); } @Test public void testDeleteTenantDeletingAllRelatedEntities() throws Exception { - TenantProfile savedProfile = createAndSaveTenantProfile(); - Tenant savedTenant = createAndSaveTenant(savedProfile); - User savedUser = createAndSaveUserFor(savedTenant); - Customer savedCustomer = createAndSaveCustomerFor(savedTenant); - WidgetsBundle savedWidgetsBundle = createAndSaveWidgetBundleFor(savedTenant); - DeviceProfile savedDeviceProfile = createAndSaveDeviceProfileWithProfileDataFor(savedTenant); - Device savedDevice = createAndSaveDeviceFor(savedTenant, savedCustomer, savedDeviceProfile); - EntityView savedEntityView = createAndSaveEntityViewFor(savedTenant, savedCustomer, savedDevice); - Asset savedAsset = createAndSaveAssetFor(savedTenant, savedCustomer); - Dashboard savedDashboard = createAndSaveDashboardFor(savedTenant, savedCustomer); - RuleChain savedRuleChain = createAndSaveRuleChainFor(savedTenant); - Edge savedEdge = createAndSaveEdgeFor(savedTenant); - OtaPackage savedOtaPackage = createAndSaveOtaPackageFor(savedTenant, savedDeviceProfile); - TbResource savedResource = createAndSaveResourceFor(savedTenant); - Rpc savedRpc = createAndSaveRpcFor(savedTenant, savedDevice); - - tenantService.deleteTenant(savedTenant.getId()); - - Assert.assertNull(tenantService.findTenantById(savedTenant.getId())); - assertCustomerIsDeleted(savedTenant, savedCustomer); - assertWidgetsBundleIsDeleted(savedTenant, savedWidgetsBundle); - assertEntityViewIsDeleted(savedTenant, savedDevice, savedEntityView); - assertAssetIsDeleted(savedTenant, savedAsset); - assertDeviceIsDeleted(savedTenant, savedDevice); - assertDeviceProfileIsDeleted(savedTenant, savedDeviceProfile); - assertDashboardIsDeleted(savedTenant, savedDashboard); - assertEdgeIsDeleted(savedTenant, savedEdge); - assertTenantAdminIsDeleted(savedTenant); - assertUserIsDeleted(savedTenant, savedUser); - Assert.assertNull(ruleChainService.findRuleChainById(savedTenant.getId(), savedRuleChain.getId())); - Assert.assertNull(apiUsageStateService.findTenantApiUsageState(savedTenant.getId())); - assertResourceIsDeleted(savedTenant, savedResource); - assertOtaPackageIsDeleted(savedTenant, savedOtaPackage); - Assert.assertNull(rpcService.findById(savedTenant.getId(), savedRpc.getId())); - - tenantProfileService.deleteTenantProfile(TenantId.SYS_TENANT_ID, savedProfile.getId()); - } - - private void assertOtaPackageIsDeleted(Tenant savedTenant, OtaPackage savedOtaPackage) { - Assert.assertNull( - otaPackageService.findOtaPackageById( - savedTenant.getId(), savedOtaPackage.getId() - ) - ); - PageLink pageLinkOta = new PageLink(1000); - PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(savedTenant.getId(), pageLinkOta); - Assert.assertFalse(pageDataOta.hasNext()); + TenantProfile profile = createAndSaveTenantProfile(); + Tenant tenant = createAndSaveTenant(profile); + User user = createAndSaveUserFor(tenant); + Customer customer = createAndSaveCustomerFor(tenant); + WidgetsBundle widgetsBundle = createAndSaveWidgetBundleFor(tenant); + DeviceProfile deviceProfile = createAndSaveDeviceProfileWithProfileDataFor(tenant); + Device device = createAndSaveDeviceFor(tenant, customer, deviceProfile); + EntityView entityView = createAndSaveEntityViewFor(tenant, customer, device); + Asset asset = createAndSaveAssetFor(tenant, customer); + Dashboard dashboard = createAndSaveDashboardFor(tenant, customer); + RuleChain ruleChain = createAndSaveRuleChainFor(tenant); + Edge edge = createAndSaveEdgeFor(tenant); + OtaPackage otaPackage = createAndSaveOtaPackageFor(tenant, deviceProfile); + TbResource resource = createAndSaveResourceFor(tenant); + Rpc rpc = createAndSaveRpcFor(tenant, device); + + tenantService.deleteTenant(tenant.getId()); + + Assert.assertNull(tenantService.findTenantById(tenant.getId())); + assertCustomerIsDeleted(tenant, customer); + assertWidgetsBundleIsDeleted(tenant, widgetsBundle); + assertEntityViewIsDeleted(tenant, device, entityView); + assertAssetIsDeleted(tenant, asset); + assertDeviceIsDeleted(tenant, device); + assertDeviceProfileIsDeleted(tenant, deviceProfile); + assertDashboardIsDeleted(tenant, dashboard); + assertEdgeIsDeleted(tenant, edge); + assertTenantAdminIsDeleted(tenant); + assertUserIsDeleted(tenant, user); + Assert.assertNull(ruleChainService.findRuleChainById(tenant.getId(), ruleChain.getId())); + Assert.assertNull(apiUsageStateService.findTenantApiUsageState(tenant.getId())); + assertResourceIsDeleted(tenant, resource); + assertOtaPackageIsDeleted(tenant, otaPackage); + Assert.assertNull(rpcService.findById(tenant.getId(), rpc.getId())); + + tenantProfileService.deleteTenantProfile(TenantId.SYS_TENANT_ID, profile.getId()); + } + + private void assertOtaPackageIsDeleted(Tenant tenant, OtaPackage otaPackage) { + assertThat(otaPackageService.findOtaPackageById(tenant.getId(), otaPackage.getId())) + .as("otaPackage").isNull(); + PageLink pageLinkOta = new PageLink(1); + PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(tenant.getId(), pageLinkOta); Assert.assertEquals(0, pageDataOta.getTotalElements()); } - private void assertResourceIsDeleted(Tenant savedTenant, TbResource savedResource) { - Assert.assertNull(resourceService.findResourceById(savedTenant.getId(), savedResource.getId())); - PageLink pageLinkResources = new PageLink(1000); + private void assertResourceIsDeleted(Tenant tenant, TbResource resource) { + assertThat(resourceService.findResourceById(tenant.getId(), resource.getId())) + .as("resource").isNull(); + PageLink pageLinkResources = new PageLink(1); PageData tenantResources = - resourceService.findAllTenantResourcesByTenantId(savedTenant.getId(), pageLinkResources); - Assert.assertFalse(tenantResources.hasNext()); + resourceService.findAllTenantResourcesByTenantId(tenant.getId(), pageLinkResources); Assert.assertEquals(0, tenantResources.getTotalElements()); } - private void assertUserIsDeleted(Tenant savedTenant, User savedUser) { - Assert.assertNull(userService.findUserById(savedTenant.getId(), savedUser.getId())); - PageLink pageLinkUsers = new PageLink(1000); + private void assertUserIsDeleted(Tenant tenant, User user) { + assertThat(userService.findUserById(tenant.getId(), user.getId())) + .as("user").isNull(); + PageLink pageLinkUsers = new PageLink(1); PageData users = - userService.findUsersByTenantId(savedTenant.getId(), pageLinkUsers); - Assert.assertFalse(users.hasNext()); + userService.findUsersByTenantId(tenant.getId(), pageLinkUsers); Assert.assertEquals(0, users.getTotalElements()); } private void assertTenantAdminIsDeleted(Tenant savedTenant) { - PageLink pageLinkTenantAdmins = new PageLink(1000); + PageLink pageLinkTenantAdmins = new PageLink(1); PageData tenantAdmins = userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); - Assert.assertFalse(tenantAdmins.hasNext()); Assert.assertEquals(0, tenantAdmins.getTotalElements()); } - private void assertEdgeIsDeleted(Tenant savedTenant, Edge savedEdge) { - Assert.assertNull(edgeService.findEdgeById(savedTenant.getId(), savedEdge.getId())); - PageLink pageLinkEdges = new PageLink(1000); - PageData edges = edgeService.findEdgesByTenantId(savedTenant.getId(), pageLinkEdges); - Assert.assertFalse(edges.hasNext()); + private void assertEdgeIsDeleted(Tenant tenant, Edge edge) { + assertThat(edgeService.findEdgeById(tenant.getId(), edge.getId())) + .as("edge").isNull(); + PageLink pageLinkEdges = new PageLink(1); + PageData edges = edgeService.findEdgesByTenantId(tenant.getId(), pageLinkEdges); Assert.assertEquals(0, edges.getTotalElements()); } - private void assertDashboardIsDeleted(Tenant savedTenant, Dashboard savedDashboard) { - Assert.assertNull(dashboardService.findDashboardById( - savedTenant.getId(), savedDashboard.getId() - )); - PageLink pageLinkDashboards = new PageLink(1000); + private void assertDashboardIsDeleted(Tenant tenant, Dashboard dashboard) { + assertThat(dashboardService.findDashboardById(tenant.getId(), dashboard.getId())) + .as("dashboard").isNull(); + PageLink pageLinkDashboards = new PageLink(1); PageData dashboards = - dashboardService.findDashboardsByTenantId(savedTenant.getId(), pageLinkDashboards); - Assert.assertFalse(dashboards.hasNext()); + dashboardService.findDashboardsByTenantId(tenant.getId(), pageLinkDashboards); Assert.assertEquals(0, dashboards.getTotalElements()); } - private void assertDeviceProfileIsDeleted(Tenant savedTenant, DeviceProfile savedDeviceProfile) { - Assert.assertNull(deviceProfileService.findDeviceProfileById( - savedTenant.getId(), savedDeviceProfile.getId() - )); - PageLink pageLinkDeviceProfiles = new PageLink(1000); + private void assertDeviceProfileIsDeleted(Tenant tenant, DeviceProfile deviceProfile) { + assertThat(deviceProfileService.findDeviceProfileById(tenant.getId(), deviceProfile.getId())) + .as("deviceProfile").isNull(); + PageLink pageLinkDeviceProfiles = new PageLink(1); PageData profiles = - deviceProfileService.findDeviceProfiles(savedTenant.getId(), pageLinkDeviceProfiles); - Assert.assertFalse(profiles.hasNext()); + deviceProfileService.findDeviceProfiles(tenant.getId(), pageLinkDeviceProfiles); Assert.assertEquals(0, profiles.getTotalElements()); } - private void assertDeviceIsDeleted(Tenant savedTenant, Device savedDevice) { - Assert.assertNull(deviceService.findDeviceById( - savedTenant.getId(), savedDevice.getId() - )); - PageLink pageLinkDevices = new PageLink(1000); + private void assertDeviceIsDeleted(Tenant tenant, Device device) { + assertThat(deviceService.findDeviceById(tenant.getId(), device.getId())) + .as("device").isNull(); + PageLink pageLinkDevices = new PageLink(1); PageData devices = - deviceService.findDevicesByTenantId(savedTenant.getId(), pageLinkDevices); - Assert.assertFalse(devices.hasNext()); + deviceService.findDevicesByTenantId(tenant.getId(), pageLinkDevices); Assert.assertEquals(0, devices.getTotalElements()); } - private void assertAssetIsDeleted(Tenant savedTenant, Asset savedAsset) { - Assert.assertNull(assetService.findAssetById( - savedTenant.getId(), savedAsset.getId() - )); - PageLink pageLinkAssets = new PageLink(1000); + private void assertAssetIsDeleted(Tenant tenant, Asset asset) { + assertThat(assetService.findAssetById(tenant.getId(), asset.getId())) + .as("asset").isNull(); + PageLink pageLinkAssets = new PageLink(1); PageData assets = - assetService.findAssetsByTenantId(savedTenant.getId(), pageLinkAssets); - Assert.assertFalse(assets.hasNext()); + assetService.findAssetsByTenantId(tenant.getId(), pageLinkAssets); Assert.assertEquals(0, assets.getTotalElements()); } - private void assertEntityViewIsDeleted(Tenant savedTenant, Device savedDevice, EntityView savedEntityView) { - Assert.assertNull(entityViewService.findEntityViewById( - savedTenant.getId(), savedEntityView.getId() - )); + private void assertEntityViewIsDeleted(Tenant tenant, Device device, EntityView entityView) { + assertThat(entityViewService.findEntityViewById(tenant.getId(), entityView.getId())) + .as("entityView").isNull(); List entityViews = - entityViewService.findEntityViewsByTenantIdAndEntityId( - savedTenant.getId(), savedDevice.getId()); + entityViewService.findEntityViewsByTenantIdAndEntityId(tenant.getId(), device.getId()); Assert.assertTrue(entityViews.isEmpty()); } - private void assertWidgetsBundleIsDeleted(Tenant savedTenant, WidgetsBundle savedWidgetsBundle) { - Assert.assertNull( - widgetsBundleService.findWidgetsBundleById(savedTenant.getId(), savedWidgetsBundle.getId()) - ); + private void assertWidgetsBundleIsDeleted(Tenant tenant, WidgetsBundle widgetsBundle) { + assertThat(widgetsBundleService.findWidgetsBundleById(tenant.getId(), widgetsBundle.getId())) + .as("widgetBundle").isNull(); List widgetsBundlesByTenantId = - widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(savedTenant.getId()); + widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(tenant.getId()); Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); } - private void assertCustomerIsDeleted(Tenant savedTenant, Customer savedCustomer) { - Assert.assertNull(customerService.findCustomerById(savedTenant.getId(), savedCustomer.getId())); - PageLink pageLinkCustomer = new PageLink(1000); + private void assertCustomerIsDeleted(Tenant tenant, Customer customer) { + assertThat(customerService.findCustomerById(tenant.getId(), customer.getId())) + .as("customer").isNull(); + PageLink pageLinkCustomer = new PageLink(1); PageData pageDataCustomer = customerService - .findCustomersByTenantId(savedTenant.getId(), pageLinkCustomer); - Assert.assertFalse(pageDataCustomer.hasNext()); + .findCustomersByTenantId(tenant.getId(), pageLinkCustomer); Assert.assertEquals(0, pageDataCustomer.getTotalElements()); } - private Rpc createAndSaveRpcFor(Tenant savedTenant, Device savedDevice) { + private Rpc createAndSaveRpcFor(Tenant tenant, Device device) { Rpc rpc = new Rpc(); - rpc.setTenantId(savedTenant.getId()); - rpc.setDeviceId(savedDevice.getId()); + rpc.setTenantId(tenant.getId()); + rpc.setDeviceId(device.getId()); rpc.setStatus(RpcStatus.QUEUED); rpc.setRequest(JacksonUtil.toJsonNode("{}")); return rpcService.save(rpc); } - private TbResource createAndSaveResourceFor(Tenant savedTenant) { + private TbResource createAndSaveResourceFor(Tenant tenant) { TbResource resource = new TbResource(); - resource.setTenantId(savedTenant.getId()); + resource.setTenantId(tenant.getId()); resource.setTitle("Test resource"); resource.setResourceType(ResourceType.LWM2M_MODEL); resource.setFileName("filename.txt"); @@ -567,47 +555,49 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { return resourceService.saveResource(resource); } - private OtaPackage createAndSaveOtaPackageFor(Tenant savedTenant, DeviceProfile savedDeviceProfile) { - OtaPackage otaPackage = createFirmware(savedTenant.getId(), savedDeviceProfile.getId()); - return otaPackageService.saveOtaPackage(otaPackage); + private OtaPackage createAndSaveOtaPackageFor(Tenant tenant, DeviceProfile deviceProfile) { + return otaPackageService.saveOtaPackage( + BaseOtaPackageServiceTest.createFirmware( + tenant.getId(), "2", deviceProfile.getId()) + ); } - private Edge createAndSaveEdgeFor(Tenant savedTenant) { - Edge edge = constructEdge(savedTenant.getId(), "Test edge", "Simple"); + private Edge createAndSaveEdgeFor(Tenant tenant) { + Edge edge = constructEdge(tenant.getId(), "Test edge", "Simple"); return edgeService.saveEdge(edge, false); } - private RuleChain createAndSaveRuleChainFor(Tenant savedTenant) { + private RuleChain createAndSaveRuleChainFor(Tenant tenant) { RuleChain ruleChain = new RuleChain(); - ruleChain.setTenantId(savedTenant.getId()); + ruleChain.setTenantId(tenant.getId()); ruleChain.setName("Test rule chain"); ruleChain.setType(RuleChainType.CORE); return ruleChainService.saveRuleChain(ruleChain); } - private Dashboard createAndSaveDashboardFor(Tenant savedTenant, Customer savedCustomer) { + private Dashboard createAndSaveDashboardFor(Tenant tenant, Customer customer) { Dashboard dashboard = new Dashboard(); - dashboard.setTenantId(savedTenant.getId()); + dashboard.setTenantId(tenant.getId()); dashboard.setTitle("Test dashboard"); - dashboard.setAssignedCustomers(Set.of(savedCustomer.toShortCustomerInfo())); + dashboard.setAssignedCustomers(Set.of(customer.toShortCustomerInfo())); return dashboardService.saveDashboard(dashboard); } - private Asset createAndSaveAssetFor(Tenant savedTenant, Customer savedCustomer) { + private Asset createAndSaveAssetFor(Tenant tenant, Customer customer) { Asset asset = new Asset(); - asset.setTenantId(savedTenant.getId()); - asset.setCustomerId(savedCustomer.getId()); + asset.setTenantId(tenant.getId()); + asset.setCustomerId(customer.getId()); asset.setType("Test asset type"); asset.setName("Test asset type"); asset.setLabel("Test asset type"); return assetService.saveAsset(asset); } - private EntityView createAndSaveEntityViewFor(Tenant savedTenant, Customer savedCustomer, Device savedDevice) { + private EntityView createAndSaveEntityViewFor(Tenant tenant, Customer customer, Device device) { EntityView entityView = new EntityView(); - entityView.setEntityId(savedDevice.getId()); - entityView.setTenantId(savedTenant.getId()); - entityView.setCustomerId(savedCustomer.getId()); + entityView.setEntityId(device.getId()); + entityView.setTenantId(tenant.getId()); + entityView.setCustomerId(customer.getId()); entityView.setType("Test type"); entityView.setName("Test entity view"); entityView.setStartTimeMs(0); @@ -615,20 +605,20 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { return entityViewService.saveEntityView(entityView); } - private Device createAndSaveDeviceFor(Tenant savedTenant, Customer savedCustomer, DeviceProfile savedDeviceProfile) { + private Device createAndSaveDeviceFor(Tenant tenant, Customer customer, DeviceProfile deviceProfile) { Device device = new Device(); - device.setCustomerId(savedCustomer.getId()); - device.setTenantId(savedTenant.getId()); + device.setCustomerId(customer.getId()); + device.setTenantId(tenant.getId()); device.setType("Test type"); device.setName("TestType"); device.setLabel("Test type"); - device.setDeviceProfileId(savedDeviceProfile.getId()); + device.setDeviceProfileId(deviceProfile.getId()); return deviceService.saveDevice(device); } - private DeviceProfile createAndSaveDeviceProfileWithProfileDataFor(Tenant savedTenant) { + private DeviceProfile createAndSaveDeviceProfileWithProfileDataFor(Tenant tenant) { DeviceProfile deviceProfile = new DeviceProfile(); - deviceProfile.setTenantId(savedTenant.getId()); + deviceProfile.setTenantId(tenant.getId()); deviceProfile.setTransportType(DeviceTransportType.MQTT); deviceProfile.setName("Test device profile"); deviceProfile.setType(DeviceProfileType.DEFAULT); @@ -638,37 +628,37 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { return deviceProfileService.saveDeviceProfile(deviceProfile); } - private WidgetsBundle createAndSaveWidgetBundleFor(Tenant savedTenant) { + private WidgetsBundle createAndSaveWidgetBundleFor(Tenant tenant) { WidgetsBundle widgetsBundle = new WidgetsBundle(); - widgetsBundle.setTenantId(savedTenant.getId()); + widgetsBundle.setTenantId(tenant.getId()); widgetsBundle.setTitle("Test widgets bundle"); widgetsBundle.setAlias("TestWidgetsBundle"); widgetsBundle.setDescription("Just a simple widgets bundle"); return widgetsBundleService.saveWidgetsBundle(widgetsBundle); } - private Customer createAndSaveCustomerFor(Tenant savedTenant) { + private Customer createAndSaveCustomerFor(Tenant tenant) { Customer customer = new Customer(); customer.setTitle("Test customer"); - customer.setTenantId(savedTenant.getId()); + customer.setTenantId(tenant.getId()); customer.setEmail("testCustomer@test.com"); return customerService.saveCustomer(customer); } - private User createAndSaveUserFor(Tenant savedTenant) { + private User createAndSaveUserFor(Tenant tenant) { User user = new User(); user.setAuthority(Authority.TENANT_ADMIN); user.setEmail("tenantAdmin@test.com"); user.setFirstName("tenantAdmin"); user.setLastName("tenantAdmin"); - user.setTenantId(savedTenant.getId()); + user.setTenantId(tenant.getId()); return userService.saveUser(user); } - private Tenant createAndSaveTenant(TenantProfile savedProfile) { + private Tenant createAndSaveTenant(TenantProfile tenantProfile) { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - tenant.setTenantProfileId(savedProfile.getId()); + tenant.setTenantProfileId(tenantProfile.getId()); return tenantService.saveTenant(tenant); } @@ -677,20 +667,4 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenantProfile.setName("Test tenant profile"); return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); } - - private OtaPackage createFirmware(TenantId tenantId, DeviceProfileId deviceProfileId) { - OtaPackage firmware = new OtaPackage(); - firmware.setTenantId(tenantId); - firmware.setDeviceProfileId(deviceProfileId); - firmware.setType(FIRMWARE); - firmware.setTitle("My firmware"); - firmware.setVersion("1"); - firmware.setFileName("filename.txt"); - firmware.setContentType("text/plain"); - firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); - firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(ByteBuffer.wrap(new byte[]{(int) 1L})); - firmware.setDataSize(1L); - return otaPackageService.saveOtaPackage(firmware); - } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java index 28eafabfce..14d6a11489 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java @@ -124,7 +124,7 @@ public class TbCreateRelationNode extends TbAbstractRelationActionNode checkRelation(TbContext ctx, SearchDirectionIds sdId, String relationType) { - return ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); } private ListenableFuture processCreateRelation(TbContext ctx, EntityContainer entityContainer, SearchDirectionIds sdId, String relationType) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java index b51c960900..116c46f9ad 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java @@ -98,7 +98,7 @@ public class TbDeleteRelationNode extends TbAbstractRelationActionNode processSingle(TbContext ctx, TbMsg msg, EntityContainer entityContainer, String relationType) { SearchDirectionIds sdId = processSingleSearchDirection(msg, entityContainer); - return Futures.transformAsync(ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), + return Futures.transformAsync(ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), result -> { if (result) { return processSingleDeleteRelation(ctx, sdId, relationType); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java index f2eb539a3e..77158fe05e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java @@ -82,7 +82,7 @@ public class TbCheckRelationNode implements TbNode { to = EntityIdFactory.getByTypeAndId(config.getEntityType(), config.getEntityId()); from = msg.getOriginator(); } - return ctx.getRelationService().checkRelation(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); } private ListenableFuture processList(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java index b1e96f228e..352bbacc23 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java @@ -113,7 +113,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -144,7 +144,7 @@ public class TbCreateRelationNodeTest { when(ctx.getRelationService().findByToAndTypeAsync(any(), eq(msg.getOriginator()), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(Collections.singletonList(relation))); when(ctx.getRelationService().deleteRelationAsync(any(), eq(relation))).thenReturn(Futures.immediateFuture(true)); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -171,7 +171,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); From c28e13c03c289ca8b54a9c9a357ee28c8a19a74f Mon Sep 17 00:00:00 2001 From: desoliture Date: Thu, 20 Jan 2022 17:09:20 +0200 Subject: [PATCH 014/178] remove todo --- .../thingsboard/server/dao/entityview/EntityViewServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 17bccd93c3..4ab8f279c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -447,7 +447,6 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti throw new DataValidationException("Entity view should be assigned to tenant!"); } else { Tenant tenant = tenantService.findTenantById(entityView.getTenantId()); - // TODO: 13.01.22 Instead of finding and checking for null need to create and use tenantService.exists() if (tenant == null) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } From fdc318a526dcd1a25e3bcb3b87abe64a6f5f6827 Mon Sep 17 00:00:00 2001 From: desoliture Date: Fri, 21 Jan 2022 11:32:05 +0200 Subject: [PATCH 015/178] fix RelationService and TenantServiceTest --- .../server/dao/relation/BaseRelationService.java | 6 ++++-- .../server/dao/service/BaseTenantServiceTest.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 1f5f4ffbe2..b1f3fda91d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -75,14 +75,16 @@ public class BaseRelationService implements RelationService { @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { - log.trace("Executing checkRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); + log.trace("Executing checkRelationAsync [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); return relationDao.checkRelationAsync(tenantId, from, to, relationType, typeGroup); } @Override public Boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { - return null; + log.trace("Executing checkRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); + validate(from, to, relationType, typeGroup); + return relationDao.checkRelation(tenantId, from, to, relationType, typeGroup); } @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #to, #relationType, #typeGroup}") diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index b36467a307..3c4009144d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -329,7 +329,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { } @Test - public void testGettingTenantAddItToCache() { + public void testGettingTenantAddingItToCache() { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); Tenant savedTenant = tenantService.saveTenant(tenant); From 6d1969447b3c8b8736a319437652d73a8e712b68 Mon Sep 17 00:00:00 2001 From: desoliture Date: Fri, 21 Jan 2022 16:55:39 +0200 Subject: [PATCH 016/178] add 'exists' method in TenantService and make it cacheable, update corresponding test --- .../server/dao/tenant/TenantService.java | 4 +- .../server/dao/tenant/TenantServiceImpl.java | 18 +++++-- .../dao/service/BaseTenantServiceTest.java | 53 ++++++++++++++++--- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java index 01c4d9b4e2..899d85993f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java @@ -31,7 +31,9 @@ public interface TenantService { ListenableFuture findTenantByIdAsync(TenantId callerId, TenantId tenantId); Tenant saveTenant(Tenant tenant); - + + boolean exists(TenantId tenantId); + void deleteTenant(TenantId tenantId); PageData findTenants(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 50fe23f22b..48c86d8241 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; +import org.springframework.cache.annotation.Caching; import org.springframework.stereotype.Service; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -107,7 +108,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private RpcService rpcService; @Override - @Cacheable(cacheNames = TENANTS_CACHE, key = "#tenantId") + @Cacheable(cacheNames = TENANTS_CACHE, key = "{#tenantId, 'TENANT'}") public Tenant findTenantById(TenantId tenantId) { log.trace("Executing findTenantById [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -130,7 +131,10 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override @Transactional - @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenant.id", condition = "#tenant.id!=null") + @Caching(evict = { + @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenant.id, 'TENANT'}", condition = "#tenant.id!=null"), + @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenant.id, 'EXISTS'}", condition = "#tenant.id!=null") + }) public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -149,7 +153,10 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override @Transactional(timeout = 60 * 60) - @CacheEvict(cacheNames = TENANTS_CACHE, key = "#tenantId") + @Caching(evict = { + @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenantId, 'TENANT'}"), + @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenantId, 'EXISTS'}") + }) public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -198,6 +205,11 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe return tenantDao.findTenantsIds(pageLink); } + @Cacheable(cacheNames = TENANTS_CACHE, key = "{#tenantId, 'EXISTS'}") + public boolean exists(TenantId tenantId) { + return tenantDao.existsById(tenantId, tenantId.getId()); + } + private DataValidator tenantValidator = new DataValidator() { @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 3c4009144d..9021f1fd8a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -335,14 +335,14 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Tenant savedTenant = tenantService.saveTenant(tenant); Mockito.reset(tenantDao); - Objects.requireNonNull(tenantCache, "Tenant cache manager is null").evict(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Tenant cache manager is null").evict(List.of(savedTenant.getId(), "TENANT")); verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedTenant); for (int i = 0; i < 100; i++) { @@ -353,6 +353,30 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenantService.deleteTenant(savedTenant.getId()); } + @Test + public void testExistsTenantAddingResultToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + + verify(tenantDao, Mockito.times(0)).existsById(any(), any()); + tenantService.exists(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + Cache.ValueWrapper cachedExists = + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "EXISTS")); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedExists); + + for (int i = 0; i < 100; i++) { + tenantService.exists(savedTenant.getId()); + } + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + @Test public void testUpdatingExistingTenantEvictCache() { Tenant tenant = new Tenant(); @@ -360,7 +384,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Tenant savedTenant = tenantService.saveTenant(tenant); Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); savedTenant.setTitle("My new tenant"); @@ -368,13 +392,19 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Mockito.reset(tenantDao); - cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); + cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "EXISTS")); Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + Mockito.reset(tenantDao); + + verify(tenantDao, Mockito.times(0)).existsById(any(), any()); + tenantService.exists(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + tenantService.deleteTenant(savedTenant.getId()); } @@ -384,13 +414,24 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTitle("My tenant"); Tenant savedTenant = tenantService.saveTenant(tenant); + tenantService.exists(savedTenant.getId()); + Cache.ValueWrapper cachedTenant = - Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); + Cache.ValueWrapper cachedExists = + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "EXISTS")); Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedExists); tenantService.deleteTenant(savedTenant.getId()); - cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(savedTenant.getId()); + cachedTenant = + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); + cachedExists = + Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "EXISTS")); + + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedExists); } @Test From 27efb4adfc48075e5f2df96c6630f39c93167d84 Mon Sep 17 00:00:00 2001 From: desoliture Date: Fri, 21 Jan 2022 18:49:35 +0200 Subject: [PATCH 017/178] replace getting tenant and checking it for null to invocations of 'exists' method --- .../org/thingsboard/server/dao/alarm/BaseAlarmService.java | 3 +-- .../org/thingsboard/server/dao/asset/BaseAssetService.java | 3 +-- .../thingsboard/server/dao/customer/CustomerServiceImpl.java | 3 +-- .../server/dao/dashboard/DashboardServiceImpl.java | 3 +-- .../server/dao/device/DeviceProfileServiceImpl.java | 3 +-- .../org/thingsboard/server/dao/device/DeviceServiceImpl.java | 3 +-- .../java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java | 3 +-- .../server/dao/entityview/EntityViewServiceImpl.java | 3 +-- .../org/thingsboard/server/dao/ota/BaseOtaPackageService.java | 4 +--- .../thingsboard/server/dao/resource/BaseResourceService.java | 3 +-- .../org/thingsboard/server/dao/rule/BaseRuleChainService.java | 3 +-- .../server/dao/usagerecord/ApiUsageStateServiceImpl.java | 3 +-- .../java/org/thingsboard/server/dao/user/UserServiceImpl.java | 4 +--- .../thingsboard/server/dao/widget/WidgetTypeServiceImpl.java | 3 +-- .../server/dao/widget/WidgetsBundleServiceImpl.java | 3 +-- 15 files changed, 15 insertions(+), 32 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index e2e1d54935..1390cdf2d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -430,8 +430,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ if (alarm.getTenantId() == null) { throw new DataValidationException("Alarm should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(alarm.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(alarm.getTenantId())) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 2610aff3e3..f1b84eac21 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -411,8 +411,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ if (asset.getTenantId() == null) { throw new DataValidationException("Asset should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(asset.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(asset.getTenantId())) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 3b2376f98d..a0ba27e8bf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -210,8 +210,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom if (customer.getTenantId() == null) { throw new DataValidationException("Customer should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(customer.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(customer.getTenantId())) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index a27d69f7ee..c04f25079f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -308,8 +308,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb if (dashboard.getTenantId() == null) { throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(dashboard.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(dashboard.getTenantId())) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 254b84889b..561f703bb9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -375,8 +375,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D if (deviceProfile.getTenantId() == null) { throw new DataValidationException("Device profile should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(deviceProfile.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(deviceProfile.getTenantId())) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 00d393c4bd..a2f289532c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -731,8 +731,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe if (device.getTenantId() == null) { throw new DataValidationException("Device should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(device.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(device.getTenantId())) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 281848a68d..5af1ffe506 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -413,8 +413,7 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic if (edge.getTenantId() == null) { throw new DataValidationException("Edge should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(edge.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(edge.getTenantId())) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 4ab8f279c3..1ac8e91bac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -446,8 +446,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti if (entityView.getTenantId() == null) { throw new DataValidationException("Entity view should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(entityView.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(entityView.getTenantId())) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 734c96b1c7..a9f1c83330 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -357,9 +357,7 @@ public class BaseOtaPackageService implements OtaPackageService { if (otaPackageInfo.getTenantId() == null) { throw new DataValidationException("OtaPackage should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(otaPackageInfo.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() - if (tenant == null) { + if (!tenantService.exists(otaPackageInfo.getTenantId())) { throw new DataValidationException("OtaPackage is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 2511a7ec6a..013d754959 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -183,8 +183,7 @@ public class BaseResourceService implements ResourceService { resource.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantService.findTenantById(resource.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(resource.getTenantId())) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1ef88f5450..1c5e0e893d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -726,8 +726,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getTenantId() == null || ruleChain.getTenantId().isNullUid()) { throw new DataValidationException("Rule chain should be assigned to tenant!"); } - Tenant tenant = tenantService.findTenantById(ruleChain.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(ruleChain.getTenantId())) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } if (ruleChain.isRoot() && RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 8cb4e17598..4635136a67 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -164,8 +164,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (apiUsageState.getTenantId() == null) { throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { - Tenant tenant = tenantService.findTenantById(apiUsageState.getTenantId()); - if (tenant == null && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { + if (!tenantService.exists(apiUsageState.getTenantId()) && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 171cb5fbd1..3032a5a8ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -448,9 +448,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic + " already present in database!"); } if (!tenantId.getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantService.findTenantById(user.getTenantId()); - // TODO: 12.01.22 Instead of finding and checking for null need to create and use tenantService.exists() - if (tenant == null) { + if (!tenantService.exists(user.getTenantId())) { throw new DataValidationException("User is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index e2fe57ca74..24f6ce905d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -138,8 +138,7 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { widgetTypeDetails.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantService.findTenantById(widgetTypeDetails.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(widgetTypeDetails.getTenantId())) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 2bea7f3ade..18900e7e28 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -162,8 +162,7 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { widgetsBundle.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantService.findTenantById(widgetsBundle.getTenantId()); - if (tenant == null) { + if (!tenantService.exists(widgetsBundle.getTenantId())) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } } From 6121f96a9cedeb367276f62f57f4ecddd6a075b7 Mon Sep 17 00:00:00 2001 From: desoliture Date: Tue, 25 Jan 2022 12:43:23 +0200 Subject: [PATCH 018/178] BaseTenantServiceTest --- .../thingsboard/server/dao/service/BaseTenantServiceTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 9021f1fd8a..96737a6c75 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -360,6 +360,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Tenant savedTenant = tenantService.saveTenant(tenant); Mockito.reset(tenantDao); + tenantCache.clear(); verify(tenantDao, Mockito.times(0)).existsById(any(), any()); tenantService.exists(savedTenant.getId()); From 54c48e1166802d8cff75cfa73bca6c6625c5cce9 Mon Sep 17 00:00:00 2001 From: deso-deso Date: Tue, 25 Jan 2022 13:35:47 +0200 Subject: [PATCH 019/178] refactor tenantExists method --- .../org/thingsboard/server/dao/tenant/TenantService.java | 2 +- .../thingsboard/server/dao/alarm/BaseAlarmService.java | 3 +-- .../thingsboard/server/dao/asset/BaseAssetService.java | 4 +--- .../server/dao/customer/CustomerServiceImpl.java | 4 +--- .../server/dao/dashboard/DashboardServiceImpl.java | 3 +-- .../server/dao/device/DeviceProfileServiceImpl.java | 3 +-- .../thingsboard/server/dao/device/DeviceServiceImpl.java | 4 +--- .../org/thingsboard/server/dao/edge/EdgeServiceImpl.java | 3 +-- .../server/dao/entityview/EntityViewServiceImpl.java | 4 +--- .../thingsboard/server/dao/ota/BaseOtaPackageService.java | 3 +-- .../server/dao/resource/BaseResourceService.java | 3 +-- .../thingsboard/server/dao/rule/BaseRuleChainService.java | 3 +-- .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 7 ++----- .../server/dao/usagerecord/ApiUsageStateServiceImpl.java | 2 +- .../org/thingsboard/server/dao/user/UserServiceImpl.java | 3 +-- .../server/dao/widget/WidgetTypeServiceImpl.java | 3 +-- .../server/dao/widget/WidgetsBundleServiceImpl.java | 3 +-- .../server/dao/service/BaseTenantServiceTest.java | 8 ++++---- 18 files changed, 22 insertions(+), 43 deletions(-) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java index 899d85993f..6a521c2b5e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java @@ -32,7 +32,7 @@ public interface TenantService { Tenant saveTenant(Tenant tenant); - boolean exists(TenantId tenantId); + boolean tenantExists(TenantId tenantId); void deleteTenant(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 1390cdf2d3..49c75914b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -27,7 +27,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -430,7 +429,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ if (alarm.getTenantId() == null) { throw new DataValidationException("Alarm should be assigned to tenant!"); } else { - if (!tenantService.exists(alarm.getTenantId())) { + if (!tenantService.tenantExists(alarm.getTenantId())) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f1b84eac21..1823f47db7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -62,7 +61,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.CacheConstants.ASSET_CACHE; @@ -411,7 +409,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ if (asset.getTenantId() == null) { throw new DataValidationException("Asset should be assigned to tenant!"); } else { - if (!tenantService.exists(asset.getTenantId())) { + if (!tenantService.tenantExists(asset.getTenantId())) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index a0ba27e8bf..dc6375f521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -25,7 +25,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -35,7 +34,6 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; @@ -210,7 +208,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom if (customer.getTenantId() == null) { throw new DataValidationException("Customer should be assigned to tenant!"); } else { - if (!tenantService.exists(customer.getTenantId())) { + if (!tenantService.tenantExists(customer.getTenantId())) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index c04f25079f..18ae67da43 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -308,7 +307,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb if (dashboard.getTenantId() == null) { throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { - if (!tenantService.exists(dashboard.getTenantId())) { + if (!tenantService.tenantExists(dashboard.getTenantId())) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 561f703bb9..60735d80ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -45,7 +45,6 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -375,7 +374,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D if (deviceProfile.getTenantId() == null) { throw new DataValidationException("Device profile should be assigned to tenant!"); } else { - if (!tenantService.exists(deviceProfile.getTenantId())) { + if (!tenantService.tenantExists(deviceProfile.getTenantId())) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index a2f289532c..66eabed0e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.device.data.CoapDeviceTransportConfiguration; @@ -91,7 +90,6 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.CacheConstants.DEVICE_CACHE; @@ -731,7 +729,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe if (device.getTenantId() == null) { throw new DataValidationException("Device should be assigned to tenant!"); } else { - if (!tenantService.exists(device.getTenantId())) { + if (!tenantService.tenantExists(device.getTenantId())) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 5af1ffe506..d90884bc59 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -34,7 +34,6 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -413,7 +412,7 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic if (edge.getTenantId() == null) { throw new DataValidationException("Edge should be assigned to tenant!"); } else { - if (!tenantService.exists(edge.getTenantId())) { + if (!tenantService.tenantExists(edge.getTenantId())) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 1ac8e91bac..9dbab008ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -34,7 +34,6 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.entityview.EntityViewSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; @@ -61,7 +60,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.CacheConstants.ENTITY_VIEW_CACHE; @@ -446,7 +444,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti if (entityView.getTenantId() == null) { throw new DataValidationException("Entity view should be assigned to tenant!"); } else { - if (!tenantService.exists(entityView.getTenantId())) { + if (!tenantService.tenantExists(entityView.getTenantId())) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index a9f1c83330..daf4aeb5b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; @@ -357,7 +356,7 @@ public class BaseOtaPackageService implements OtaPackageService { if (otaPackageInfo.getTenantId() == null) { throw new DataValidationException("OtaPackage should be assigned to tenant!"); } else { - if (!tenantService.exists(otaPackageInfo.getTenantId())) { + if (!tenantService.tenantExists(otaPackageInfo.getTenantId())) { throw new DataValidationException("OtaPackage is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 013d754959..6d3d29540b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -24,7 +24,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -183,7 +182,7 @@ public class BaseResourceService implements ResourceService { resource.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - if (!tenantService.exists(resource.getTenantId())) { + if (!tenantService.tenantExists(resource.getTenantId())) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 1c5e0e893d..9b01af1832 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -30,7 +30,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; @@ -726,7 +725,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getTenantId() == null || ruleChain.getTenantId().isNullUid()) { throw new DataValidationException("Rule chain should be assigned to tenant!"); } - if (!tenantService.exists(ruleChain.getTenantId())) { + if (!tenantService.tenantExists(ruleChain.getTenantId())) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } if (ruleChain.isRoot() && RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 48c86d8241..6d685bceb9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -131,10 +131,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override @Transactional - @Caching(evict = { - @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenant.id, 'TENANT'}", condition = "#tenant.id!=null"), - @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenant.id, 'EXISTS'}", condition = "#tenant.id!=null") - }) + @CacheEvict(cacheNames = TENANTS_CACHE, key = "{#tenant.id, 'TENANT'}", condition = "#tenant.id!=null") public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -206,7 +203,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Cacheable(cacheNames = TENANTS_CACHE, key = "{#tenantId, 'EXISTS'}") - public boolean exists(TenantId tenantId) { + public boolean tenantExists(TenantId tenantId) { return tenantDao.existsById(tenantId, tenantId.getId()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 4635136a67..a9864b1e1e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -164,7 +164,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (apiUsageState.getTenantId() == null) { throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { - if (!tenantService.exists(apiUsageState.getTenantId()) && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { + if (!tenantService.tenantExists(apiUsageState.getTenantId()) && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 3032a5a8ad..6647958965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -31,7 +31,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -448,7 +447,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic + " already present in database!"); } if (!tenantId.getId().equals(ModelConstants.NULL_UUID)) { - if (!tenantService.exists(user.getTenantId())) { + if (!tenantService.tenantExists(user.getTenantId())) { throw new DataValidationException("User is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 24f6ce905d..42195fdfda 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.widget.WidgetType; @@ -138,7 +137,7 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { widgetTypeDetails.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - if (!tenantService.exists(widgetTypeDetails.getTenantId())) { + if (!tenantService.tenantExists(widgetTypeDetails.getTenantId())) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 18900e7e28..0a9c23c972 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; @@ -162,7 +161,7 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { widgetsBundle.setTenantId(new TenantId(ModelConstants.NULL_UUID)); } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - if (!tenantService.exists(widgetsBundle.getTenantId())) { + if (!tenantService.tenantExists(widgetsBundle.getTenantId())) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 96737a6c75..c48bdd1dcf 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -363,7 +363,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenantCache.clear(); verify(tenantDao, Mockito.times(0)).existsById(any(), any()); - tenantService.exists(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); Cache.ValueWrapper cachedExists = @@ -371,7 +371,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedExists); for (int i = 0; i < 100; i++) { - tenantService.exists(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); } verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); @@ -403,7 +403,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Mockito.reset(tenantDao); verify(tenantDao, Mockito.times(0)).existsById(any(), any()); - tenantService.exists(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); tenantService.deleteTenant(savedTenant.getId()); @@ -415,7 +415,7 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTitle("My tenant"); Tenant savedTenant = tenantService.saveTenant(tenant); - tenantService.exists(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); Cache.ValueWrapper cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); From 80df8d1a4583d5f001044a42278004e7d7c82b55 Mon Sep 17 00:00:00 2001 From: deso-deso Date: Tue, 25 Jan 2022 18:52:48 +0200 Subject: [PATCH 020/178] fix test --- .../server/dao/service/BaseTenantServiceTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index c48bdd1dcf..216d23d71f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -393,19 +393,13 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { Mockito.reset(tenantDao); - cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "EXISTS")); + cachedTenant = Objects.requireNonNull(tenantCache, "Cache manager is null!").get(List.of(savedTenant.getId(), "TENANT")); Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); verify(tenantDao, Mockito.times(0)).findById(any(), any()); tenantService.findTenantById(savedTenant.getId()); verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); - Mockito.reset(tenantDao); - - verify(tenantDao, Mockito.times(0)).existsById(any(), any()); - tenantService.tenantExists(savedTenant.getId()); - verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); - tenantService.deleteTenant(savedTenant.getId()); } From 4062dc70fb196e6b85c4ada990d541c01aab6875 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 15:03:55 +0200 Subject: [PATCH 021/178] Initial export/import API for devices, device profiles, assets, rule chains, dashboards and customers --- .../main/data/upgrade/3.4/schema_update.sql | 28 +++ .../EntitiesExportImportController.java | 99 +++++++++ .../DefaultEntitiesExportImportService.java | 86 ++++++++ .../expimp/EntitiesExportImportService.java | 29 +++ .../expimp/exp/EntityExportService.java | 32 +++ .../expimp/exp/impl/AssetExportService.java | 50 +++++ .../exp/impl/CustomerExportService.java | 50 +++++ .../exp/impl/DashboardExportService.java | 50 +++++ .../expimp/exp/impl/DeviceExportService.java | 53 +++++ .../exp/impl/DeviceProfileExportService.java | 50 +++++ .../exp/impl/RuleChainExportService.java | 51 +++++ .../expimp/imp/EntityImportService.java | 30 +++ .../imp/impl/AbstractEntityImportService.java | 73 +++++++ .../expimp/imp/impl/AssetImportService.java | 62 ++++++ .../imp/impl/CustomerImportService.java | 60 ++++++ .../imp/impl/DashboardImportService.java | 62 ++++++ .../expimp/imp/impl/DeviceImportService.java | 81 ++++++++ .../imp/impl/DeviceProfileImportService.java | 65 ++++++ .../imp/impl/RuleChainImportService.java | 80 ++++++++ .../server/dao/rule/RuleChainService.java | 3 +- .../server/common/data/Customer.java | 6 + .../server/common/data/Dashboard.java | 10 +- .../server/common/data/Device.java | 7 + .../server/common/data/DeviceProfile.java | 3 + .../server/common/data/asset/Asset.java | 7 + .../data/export/EntitiesExportRequest.java | 29 +++ .../data/export/EntitiesExportResponse.java | 29 +++ .../common/data/export/EntityExportData.java | 48 +++++ .../data/export/impl/AssetExportData.java | 33 +++ .../data/export/impl/CustomerExportData.java | 33 +++ .../data/export/impl/DashboardExportData.java | 33 +++ .../data/export/impl/DeviceExportData.java | 35 ++++ .../export/impl/DeviceProfileExportData.java | 33 +++ .../data/export/impl/RuleChainExportData.java | 35 ++++ .../server/common/data/rule/RuleChain.java | 3 + .../server/dao/ExportableEntityDao.java | 36 ++++ .../dao/ExportableEntityRepository.java | 26 +++ .../server/dao/asset/AssetDao.java | 4 +- .../server/dao/customer/CustomerDao.java | 5 +- .../server/dao/dashboard/DashboardDao.java | 3 +- .../server/dao/device/DeviceDao.java | 3 +- .../server/dao/device/DeviceProfileDao.java | 3 +- .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AbstractAssetEntity.java | 11 + .../dao/model/sql/AbstractDeviceEntity.java | 10 + .../server/dao/model/sql/CustomerEntity.java | 9 + .../server/dao/model/sql/DashboardEntity.java | 9 + .../dao/model/sql/DeviceProfileEntity.java | 9 + .../server/dao/model/sql/RuleChainEntity.java | 9 + .../server/dao/rule/BaseRuleChainService.java | 5 + .../server/dao/rule/RuleChainDao.java | 7 +- .../server/dao/sql/asset/AssetRepository.java | 3 +- .../server/dao/sql/asset/JpaAssetDao.java | 16 ++ .../dao/sql/customer/CustomerRepository.java | 3 +- .../dao/sql/customer/JpaCustomerDao.java | 17 ++ .../sql/dashboard/DashboardRepository.java | 4 +- .../dao/sql/dashboard/JpaDashboardDao.java | 18 ++ .../sql/device/DeviceProfileRepository.java | 4 +- .../dao/sql/device/DeviceRepository.java | 3 +- .../server/dao/sql/device/JpaDeviceDao.java | 16 ++ .../dao/sql/device/JpaDeviceProfileDao.java | 17 ++ .../server/dao/sql/rule/JpaRuleChainDao.java | 16 ++ .../dao/sql/rule/RuleChainRepository.java | 3 +- .../main/resources/sql/schema-entities.sql | 188 +++++++++--------- 64 files changed, 1788 insertions(+), 109 deletions(-) create mode 100644 application/src/main/data/upgrade/3.4/schema_update.sql create mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java diff --git a/application/src/main/data/upgrade/3.4/schema_update.sql b/application/src/main/data/upgrade/3.4/schema_update.sql new file mode 100644 index 0000000000..ad307a6576 --- /dev/null +++ b/application/src/main/data/upgrade/3.4/schema_update.sql @@ -0,0 +1,28 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER TABLE device + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE device_profile + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE asset + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_chain + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE dashboard + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE customer + ADD COLUMN IF NOT EXISTS external_id UUID; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java new file mode 100644 index 0000000000..a11a9189f4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.export.EntitiesExportRequest; +import org.thingsboard.server.common.data.export.EntitiesExportResponse; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.EntitiesExportImportService; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@RestController +@RequestMapping("/api/entities") +@TbCoreComponent +@RequiredArgsConstructor +public class EntitiesExportImportController extends BaseController { + + private final EntitiesExportImportService exportImportService; + + + @PostMapping("/export/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData exportEntity(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + try { + return exportImportService.exportEntity(getTenantId(), entityId); + } catch (Exception e) { + throw handleException(e); + } + } + + @PostMapping("/export/batch") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { + TenantId tenantId = getTenantId(); + + EntitiesExportResponse exportResponse = new EntitiesExportResponse(); + + Map>>> result = new HashMap<>(); + exportRequest.getEntities().forEach((entityType, entityIds) -> { + List>> exportDataForEntityType = new LinkedList<>(); + entityIds.forEach(entityId -> { + EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); + exportDataForEntityType.add(exportData); + }); + result.put(entityType, exportDataForEntityType); + }); + + exportResponse.setExportData(result); + return exportResponse; + } + // TODO: api to export and import whole customer, whole tenant + + + @PostMapping("/import") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public , I extends EntityId, D extends EntityExportData> E importEntity(@RequestBody D exportData) throws ThingsboardException { + try { + return exportImportService.importEntity(getTenantId(), exportData); + } catch (Exception e) { + throw handleException(e); + } + } + +// public void importEntities(@RequestBody ) + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java new file mode 100644 index 0000000000..9b384b34dd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.imp.EntityImportService; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +// FIXME: review packages and classes naming +@Service +@TbCoreComponent +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { + + private final Map> exportServices = new EnumMap<>(EntityType.class); + private final Map> importServices = new EnumMap<>(EntityType.class); + + + // TODO: export and import of the whole tenant + // TODO: export and import of the whole customer ? + // TODO: relations export and import + @Override + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + EntityType entityType = entityId.getEntityType(); + EntityExportService exportService = getExportService(entityType); + + return exportService.getExportData(tenantId, entityId); + } + + // FIXME: somehow validate export data + // FIXME: validate permissions for create or update + // FIXME: send entity lifecycle event + @Override + public , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData) { + EntityType entityType = exportData.getEntityType(); + EntityImportService importService = getImportService(entityType); + + return importService.importEntity(tenantId, exportData); + } + + + @SuppressWarnings("unchecked") + private > EntityExportService getExportService(EntityType entityType) { + return (EntityExportService) exportServices.get(entityType); + } + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + return (EntityImportService) importServices.get(entityType); + } + + @Autowired + private void setExportImportServices(Collection> exportServices, + Collection> importServices) { + exportServices.forEach(entityExportService -> { + this.exportServices.put(entityExportService.getEntityType(), entityExportService); + }); + importServices.forEach(entityImportService -> { + this.importServices.put(entityImportService.getEntityType(), entityImportService); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java new file mode 100644 index 0000000000..5d25cb9320 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntitiesExportImportService { + + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + + , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java new file mode 100644 index 0000000000..9afa01ff8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntityExportService> { + + // FIXME: export relations + // FIXME: get rid of boilerplate + EntityExportData getExportData(TenantId tenantId, I entityId); + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java new file mode 100644 index 0000000000..bb4b68bb29 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetExportService implements EntityExportService { + + private final AssetService assetService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, AssetId assetId) { + AssetExportData exportData = new AssetExportData(); + exportData.setAsset(assetService.findAssetById(tenantId, assetId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java new file mode 100644 index 0000000000..7430f9ddd3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerExportService implements EntityExportService { + + private final CustomerService customerService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, CustomerId customerId) { + CustomerExportData exportData = new CustomerExportData(); + exportData.setCustomer(customerService.findCustomerById(tenantId, customerId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java new file mode 100644 index 0000000000..87959429d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardExportService implements EntityExportService { + + private final DashboardService dashboardService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DashboardId dashboardId) { + DashboardExportData exportData = new DashboardExportData(); + exportData.setDashboard(dashboardService.findDashboardById(tenantId, dashboardId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java new file mode 100644 index 0000000000..860aaa9b1d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceExportService implements EntityExportService { + + private final DeviceService deviceService; + private final DeviceCredentialsService deviceCredentialsService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DeviceId deviceId) { + DeviceExportData exportData = new DeviceExportData(); + exportData.setDevice(deviceService.findDeviceById(tenantId, deviceId)); + exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java new file mode 100644 index 0000000000..425dbfd5be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileExportService implements EntityExportService { + + private final DeviceProfileService deviceProfileService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DeviceProfileId deviceProfileId) { + DeviceProfileExportData exportData = new DeviceProfileExportData(); + exportData.setDeviceProfile(deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java new file mode 100644 index 0000000000..7a85281d63 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainExportService implements EntityExportService { + + private final RuleChainService ruleChainService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, RuleChainId ruleChainId) { + RuleChainExportData exportData = new RuleChainExportData(); + exportData.setRuleChain(ruleChainService.findRuleChainById(tenantId, ruleChainId)); + exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java new file mode 100644 index 0000000000..195ace0b66 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntityImportService, D extends EntityExportData> { + + E importEntity(TenantId tenantId, D exportData); + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java new file mode 100644 index 0000000000..3f33ba04f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.service.expimp.imp.EntityImportService; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; + +public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { + + private final Map> daos = new EnumMap<>(EntityType.class); + + + public final E findByExternalId(TenantId tenantId, I externalId) { + return findByExternalOrInternalId(tenantId, externalId); + } + + + protected final ID getInternalId(TenantId tenantId, ID externalId) { + if (externalId == null) { + return null; + } + HasId entity = findByExternalOrInternalId(tenantId, externalId); + if (entity == null) { + throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); + } + return entity.getId(); + } + + private , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { + ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); + return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) + .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); + } + + + @SuppressWarnings("unchecked") + private ExportableEntityDao getDao(EntityType entityType) { + return (ExportableEntityDao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> this.daos.put(dao.getEntityType(), dao)); + if (!this.daos.containsKey(getEntityType())) { + throw new IllegalStateException(getClass().getSimpleName() + " requires ExportableEntityDao for entity type " + getEntityType()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java new file mode 100644 index 0000000000..5e563762d6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetImportService extends AbstractEntityImportService { + + private final AssetService assetService; + + + @Override + public Asset importEntity(TenantId tenantId, AssetExportData exportData) { + Asset asset = exportData.getAsset(); + Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... + + asset.setExternalId(asset.getId()); + asset.setTenantId(tenantId); + + if (existingAsset == null) { + asset.setId(null); + } else { + asset.setId(existingAsset.getId()); + } + + asset.setCustomerId(getInternalId(tenantId, asset.getCustomerId())); + + Asset savedAsset = assetService.saveAsset(asset); + + return savedAsset; + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java new file mode 100644 index 0000000000..9510cc0700 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerImportService extends AbstractEntityImportService { + + private final CustomerService customerService; + + + @Override + public Customer importEntity(TenantId tenantId, CustomerExportData exportData) { + Customer customer = exportData.getCustomer(); + Customer existingCustomer = findByExternalId(tenantId, customer.getId()); + + customer.setExternalId(customer.getId()); + customer.setTenantId(tenantId); + + if (existingCustomer == null) { + customer.setId(null); + } else { + customer.setId(existingCustomer.getId()); + } + + Customer savedCustomer = customerService.saveCustomer(customer); + + return savedCustomer; + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java new file mode 100644 index 0000000000..eab90bb294 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardImportService extends AbstractEntityImportService { + + private final DashboardService dashboardService; + + + @Override + public Dashboard importEntity(TenantId tenantId, DashboardExportData exportData) { + Dashboard dashboard = exportData.getDashboard(); + Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); + + dashboard.setExternalId(dashboard.getId()); + dashboard.setTenantId(tenantId); + + if (existingDashboard == null) { + dashboard.setId(null); + dashboard.setAssignedCustomers(null); // FIXME: need to assign dashboard to customers ? + } else { + dashboard.setId(existingDashboard.getId()); + dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); // we left them untouched (FIXME) + } + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + + return savedDashboard; + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java new file mode 100644 index 0000000000..052fa3e1b3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.action.EntityActionService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceImportService extends AbstractEntityImportService { + + private final DeviceService deviceService; + + + @Transactional + @Override + public Device importEntity(TenantId tenantId, DeviceExportData exportData) { + Device device = exportData.getDevice(); + Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! + // what if exporting and importing back already exported entity ? (save version and then load it back) + /* + * export entity -> id from env1 -> import this entity -> ... + * + * maybe find not only by external id but by internal too ? but then what if we will try + * */ + + device.setExternalId(device.getId()); + device.setTenantId(tenantId); + + if (existingDevice == null) { + device.setId(null); + device.setCustomerId(null); // FIXME: find and set customer + } else { + device.setId(existingDevice.getId()); + device.setCustomerId(existingDevice.getCustomerId()); + } + + // TODO or maybe set as additional config whether to update related entities when device already exists ? + // TODO: or also whether to ignore not found internal ids + + // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants + + device.setDeviceProfileId(getInternalId(tenantId, device.getDeviceProfileId())); + device.setFirmwareId(getInternalId(tenantId, device.getFirmwareId())); + device.setSoftwareId(getInternalId(tenantId, device.getSoftwareId())); + + Device savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); + + return savedDevice; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java new file mode 100644 index 0000000000..615f62d389 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileImportService extends AbstractEntityImportService { + + private final DeviceProfileService deviceProfileService; + + + @Override + public DeviceProfile importEntity(TenantId tenantId, DeviceProfileExportData exportData) { + DeviceProfile deviceProfile = exportData.getDeviceProfile(); + DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); + + deviceProfile.setExternalId(deviceProfile.getId()); + deviceProfile.setTenantId(tenantId); + + if (existingDeviceProfile == null) { + deviceProfile.setId(null); + } else { + deviceProfile.setId(existingDeviceProfile.getId()); + } + + deviceProfile.setDefaultRuleChainId(getInternalId(tenantId, deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(getInternalId(tenantId, deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(getInternalId(tenantId, deviceProfile.getFirmwareId())); + deviceProfile.setSoftwareId(getInternalId(tenantId, deviceProfile.getSoftwareId())); + + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + return savedDeviceProfile; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java new file mode 100644 index 0000000000..eb72f58a7f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainImportService extends AbstractEntityImportService { + + private final RuleChainService ruleChainService; + + + @Transactional + @Override + public RuleChain importEntity(TenantId tenantId, RuleChainExportData exportData) { + RuleChain ruleChain = exportData.getRuleChain(); + RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); + + ruleChain.setExternalId(ruleChain.getId()); + ruleChain.setTenantId(tenantId); + ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting + + if (existingRuleChain == null) { + ruleChain.setId(null); + } else { + ruleChain.setId(existingRuleChain.getId()); + } + + RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); + + if (ruleChain.getId() != null) { + ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); + } + RuleChainMetaData metaData = exportData.getMetaData(); + metaData.setRuleChainId(savedRuleChain.getId()); + metaData.getNodes().forEach(ruleNode -> { + ruleNode.setId(null); + ruleNode.setRuleChainId(null); + }); + metaData.getRuleChainConnections().forEach(ruleChainConnectionInfo -> { +// ruleChainConnectionInfo.setTargetRuleChainId(); + // TODO: check if this thing is needed for "Other Rule Chain Node" + // TODO: and check import of tenant rule chains + }); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return savedRuleChain; + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 5089a6286e..0c42b8aec8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; import org.thingsboard.server.common.data.rule.RuleChainImportResult; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; @@ -93,4 +92,6 @@ public interface RuleChainService { List findRuleNodesByTenantIdAndType(TenantId tenantId, String name, String toString); RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode); + + void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index ab21d70795..065e0c5db5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; @@ -36,6 +38,9 @@ public class Customer extends ContactBased implements HasTenantId { @ApiModelProperty(position = 5, required = true, value = "JSON object with Tenant Id") private TenantId tenantId; + @Getter @Setter + private CustomerId externalId; // FIXME: add to hashcode, equals, etc + public Customer() { super(); } @@ -48,6 +53,7 @@ public class Customer extends ContactBased implements HasTenantId { super(customer); this.tenantId = customer.getTenantId(); this.title = customer.getTitle(); + this.externalId = customer.getExternalId(); } public TenantId getTenantId() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 450c88cbb5..4b69534bc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,14 +17,19 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; public class Dashboard extends DashboardInfo { private static final long serialVersionUID = 872682138346187503L; - + private transient JsonNode configuration; - + + @Getter @Setter + private DashboardId externalId; + public Dashboard() { super(); } @@ -40,6 +45,7 @@ public class Dashboard extends DashboardInfo { public Dashboard(Dashboard dashboard) { super(dashboard); this.configuration = dashboard.getConfiguration(); + this.externalId = dashboard.getExternalId(); } @ApiModelProperty(position = 9, value = "JSON object with main configuration of the dashboard: layouts, widgets, aliases, etc. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 4d7bf5e822..519cffc39a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.id.CustomerId; @@ -61,6 +63,9 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen private OtaPackageId firmwareId; private OtaPackageId softwareId; + @Getter @Setter + private DeviceId externalId; + public Device() { super(); } @@ -80,6 +85,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setDeviceData(device.getDeviceData()); this.firmwareId = device.getFirmwareId(); this.softwareId = device.getSoftwareId(); + this.externalId = device.getExternalId(); } public Device updateDevice(Device device) { @@ -93,6 +99,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setFirmwareId(device.getFirmwareId()); this.setSoftwareId(device.getSoftwareId()); Optional.ofNullable(device.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.setExternalId(device.getExternalId()); return this; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 1dd6a2ad19..6161cd6a5c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -90,6 +90,8 @@ public class DeviceProfile extends SearchTextBased implements H @ApiModelProperty(position = 10, value = "Reference to the software OTA package. If present, the specified package will be used as default device software. ") private OtaPackageId softwareId; + private DeviceProfileId externalId; + public DeviceProfile() { super(); } @@ -112,6 +114,7 @@ public class DeviceProfile extends SearchTextBased implements H this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); this.firmwareId = deviceProfile.getFirmwareId(); this.softwareId = deviceProfile.getSoftwareId(); + this.externalId = deviceProfile.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the device profile Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index db68df4901..e0924c63ee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -49,6 +51,9 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements @Length(fieldName = "label") private String label; + @Getter @Setter + private AssetId externalId; + public Asset() { super(); } @@ -64,6 +69,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.name = asset.getName(); this.type = asset.getType(); this.label = asset.getLabel(); + this.externalId = asset.getExternalId(); } public void update(Asset asset) { @@ -73,6 +79,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.type = asset.getType(); this.label = asset.getLabel(); Optional.ofNullable(asset.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.externalId = asset.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the asset Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java new file mode 100644 index 0000000000..d3502695c0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; + +@Data +public class EntitiesExportRequest { + // todo: should be processed in a specific order, e.g. device profiles, then devices, then relations, etc. + private Map> entities; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java new file mode 100644 index 0000000000..cab75313d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +import java.util.List; +import java.util.Map; + +@Data +public class EntitiesExportResponse { + private Map>>> exportData; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java new file mode 100644 index 0000000000..36d3cb4b0c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({ + @Type(name = "DEVICE", value = DeviceExportData.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), + @Type(name = "ASSET", value = AssetExportData.class), + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), + @Type(name = "DASHBOARD", value = DashboardExportData.class), + @Type(name = "CUSTOMER", value = CustomerExportData.class) +}) +public interface EntityExportData> { + + @JsonIgnore + EntityType getEntityType(); // fixme: maybe remove if not needed, as well as generic + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java new file mode 100644 index 0000000000..c0f417bd76 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class AssetExportData implements EntityExportData { + + private Asset asset; + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java new file mode 100644 index 0000000000..31e53db223 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class CustomerExportData implements EntityExportData { + + private Customer customer; + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java new file mode 100644 index 0000000000..7860c6c553 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class DashboardExportData implements EntityExportData { + + private Dashboard dashboard; + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java new file mode 100644 index 0000000000..05f3299d02 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +@Data +public class DeviceExportData implements EntityExportData { + + private Device device; + private DeviceCredentials credentials; + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java new file mode 100644 index 0000000000..c030d0b86e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class DeviceProfileExportData implements EntityExportData { + + private DeviceProfile deviceProfile; + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java new file mode 100644 index 0000000000..11fbace911 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; + +@Data +public class RuleChainExportData implements EntityExportData { + + private RuleChain ruleChain; + private RuleChainMetaData metaData; + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 17d7ffef0b..7ff2b0dc2b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -59,6 +59,8 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @JsonIgnore private byte[] configurationBytes; + private RuleChainId externalId; + public RuleChain() { super(); } @@ -75,6 +77,7 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im this.firstRuleNodeId = ruleChain.getFirstRuleNodeId(); this.root = ruleChain.isRoot(); this.setConfiguration(ruleChain.getConfiguration()); + this.setExternalId(ruleChain.getExternalId()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java new file mode 100644 index 0000000000..c7741ec013 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public interface ExportableEntityDao { + + T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + T findByTenantIdAndId(UUID tenantId, UUID id); + // fixme: get rid of boilerplate ? + + EntityType getEntityType(); + + /* + * default > ExportableEntityRepository getExportableEntityRepository() { + * ((ExportableEntityRepository) getJpaRepository).find... + * */ + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java new file mode 100644 index 0000000000..b6a303a941 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface ExportableEntityRepository { + + D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + D findByTenantIdAndId(UUID tenantId, UUID id); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index e7533cc736..495886bcc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -22,8 +22,8 @@ import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index d79537e564..89672d2a73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Optional; @@ -28,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object @@ -37,7 +38,7 @@ public interface CustomerDao extends Dao, TenantEntityDao { * @return saved customer object */ Customer save(TenantId tenantId, Customer customer); - + /** * Find customers by tenant id and page link. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index dfe9152484..90353f49b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -18,12 +18,13 @@ package org.thingsboard.server.dao.dashboard; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 85f66e4b67..90f2b218a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -35,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index bf61cbdce5..b79cbb05ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -21,10 +21,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; -public interface DeviceProfileDao extends Dao { +public interface DeviceProfileDao extends Dao, ExportableEntityDao { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, UUID deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 8f692d5a12..0b74e79f2b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -559,6 +559,8 @@ public class ModelConstants { public static final String EDGE_EVENT_BY_ID_VIEW_NAME = "edge_event_by_id"; + public static final String EXTERNAL_ID_PROPERTY = "external_id"; + /** * Cassandra attributes and timeseries constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index 4405768f7c..0936ca2855 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -38,6 +38,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ASSET_LABEL_PROPER import static org.thingsboard.server.dao.model.ModelConstants.ASSET_NAME_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY; @Data @@ -68,6 +69,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity @Column(name = ModelConstants.ASSET_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = EXTERNAL_ID_PROPERTY) + private UUID externalId; + public AbstractAssetEntity() { super(); } @@ -87,6 +91,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.type = asset.getType(); this.label = asset.getLabel(); this.additionalInfo = asset.getAdditionalInfo(); + if (asset.getExternalId() != null) { + this.externalId = asset.getExternalId().getId(); + } } public AbstractAssetEntity(AssetEntity assetEntity) { @@ -99,6 +106,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.label = assetEntity.getLabel(); this.searchText = assetEntity.getSearchText(); this.additionalInfo = assetEntity.getAdditionalInfo(); + this.externalId = assetEntity.getExternalId(); } @Override @@ -128,6 +136,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity asset.setType(type); asset.setLabel(label); asset.setAdditionalInfo(additionalInfo); + if (externalId != null) { + asset.setExternalId(new AssetId(externalId)); + } return asset; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 717ba1f9e3..dbc0a057ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -84,6 +84,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti @Column(name = ModelConstants.DEVICE_DEVICE_DATA_PROPERTY, columnDefinition = "jsonb") private JsonNode deviceData; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY, columnDefinition = "uuid") + private UUID externalId; + public AbstractDeviceEntity() { super(); } @@ -113,6 +116,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.type = device.getType(); this.label = device.getLabel(); this.additionalInfo = device.getAdditionalInfo(); + if (device.getExternalId() != null) { + this.externalId = device.getExternalId().getId(); + } } public AbstractDeviceEntity(DeviceEntity deviceEntity) { @@ -129,6 +135,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.additionalInfo = deviceEntity.getAdditionalInfo(); this.firmwareId = deviceEntity.getFirmwareId(); this.softwareId = deviceEntity.getSoftwareId(); + this.externalId = deviceEntity.getExternalId(); } @Override @@ -164,6 +171,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti device.setType(type); device.setLabel(label); device.setAdditionalInfo(additionalInfo); + if (externalId != null) { + device.setExternalId(new DeviceId(externalId)); + } return device; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java index 61b21ecc4f..3dec26afd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java @@ -77,6 +77,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Column(name = ModelConstants.CUSTOMER_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public CustomerEntity() { super(); } @@ -97,6 +100,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea this.phone = customer.getPhone(); this.email = customer.getEmail(); this.additionalInfo = customer.getAdditionalInfo(); + if (customer.getExternalId() != null) { + this.externalId = customer.getExternalId().getId(); + } } @Override @@ -124,6 +130,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea customer.setPhone(phone); customer.setEmail(email); customer.setAdditionalInfo(additionalInfo); + if (externalId != null) { + customer.setExternalId(new CustomerId(externalId)); + } return customer; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 45b4bf9210..ccaea5524e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -78,6 +78,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Column(name = ModelConstants.DASHBOARD_CONFIGURATION_PROPERTY) private JsonNode configuration; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DashboardEntity() { super(); } @@ -102,6 +105,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S this.mobileHide = dashboard.isMobileHide(); this.mobileOrder = dashboard.getMobileOrder(); this.configuration = dashboard.getConfiguration(); + if (dashboard.getExternalId() != null) { + this.externalId = dashboard.getExternalId().getId(); + } } @Override @@ -133,6 +139,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S dashboard.setMobileHide(mobileHide); dashboard.setMobileOrder(mobileOrder); dashboard.setConfiguration(configuration); + if (externalId != null) { + dashboard.setExternalId(new DashboardId(externalId)); + } return dashboard; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index bd214a6cbc..6894bd08d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -103,6 +103,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl @Column(name = ModelConstants.DEVICE_PROFILE_SOFTWARE_ID_PROPERTY) private UUID softwareId; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DeviceProfileEntity() { super(); } @@ -137,6 +140,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (deviceProfile.getSoftwareId() != null) { this.softwareId = deviceProfile.getSoftwareId().getId(); } + if (deviceProfile.getExternalId() != null) { + this.externalId = deviceProfile.getExternalId().getId(); + } } @Override @@ -184,6 +190,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (softwareId != null) { deviceProfile.setSoftwareId(new OtaPackageId(softwareId)); } + if (externalId != null) { + deviceProfile.setExternalId(new DeviceProfileId(externalId)); + } return deviceProfile; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java index 09ed75b5e5..88c8c79fdc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java @@ -75,6 +75,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleChainEntity() { } @@ -94,6 +97,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT this.debugMode = ruleChain.isDebugMode(); this.configuration = ruleChain.getConfiguration(); this.additionalInfo = ruleChain.getAdditionalInfo(); + if (ruleChain.getExternalId() != null) { + this.externalId = ruleChain.getExternalId().getId(); + } } @Override @@ -120,6 +126,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT ruleChain.setDebugMode(debugMode); ruleChain.setConfiguration(configuration); ruleChain.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleChain.setExternalId(new RuleChainId(externalId)); + } return ruleChain; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 097779f33e..fd84c12a1b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -684,6 +684,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC throw t; } } + deleteRuleNodes(tenantId, ruleChainId); + } + + @Override + public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 542a487949..59f287eadd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -19,19 +19,18 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Collection; -import java.util.List; import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. @@ -81,4 +80,6 @@ public interface RuleChainDao extends Dao, TenantEntityDao { Collection findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name); + RuleChain findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 833fcc2de8..140ae3af86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -public interface AssetRepository extends JpaRepository { +public interface AssetRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + "FROM AssetEntity a " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index c9752c5390..cd4fefac48 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -208,4 +208,20 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); } + + @Override + public Asset findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Asset findByTenantIdAndId(UUID tenantId, UUID id) { + return DaoUtil.getData(assetRepository.findByTenantIdAndId(tenantId, id)); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 61cd9afac7..1300fcdfa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; import java.util.UUID; @@ -27,7 +28,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface CustomerRepository extends JpaRepository { +public interface CustomerRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + "AND LOWER(c.searchText) LIKE LOWER(CONCAT('%', :textSearch, '%'))") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 02311df61c..9cf74f1f33 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -68,4 +69,20 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao { +public interface DashboardRepository extends JpaRepository, ExportableEntityRepository { Long countByTenantId(UUID tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 98edcaa1fd..b65e9997a9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; @@ -49,4 +51,20 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao { +public interface DeviceProfileRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " + "FROM DeviceProfileEntity d " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index b4ae9270e3..46d755195e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -30,7 +31,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface DeviceRepository extends JpaRepository { +public interface DeviceRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo, p.name) " + "FROM DeviceEntity d " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 1d75690e50..1a5d44900c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -302,4 +302,20 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao Objects.toString(pageLink.getTextSearch(), ""), DaoUtil.toPageable(pageLink))); } + + @Override + public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Device findByTenantIdAndId(UUID tenantId, UUID id) { + return findDeviceByTenantIdAndId(TenantId.fromUUID(tenantId), id); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 80513d77c6..539377a647 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -23,6 +23,7 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -109,4 +110,20 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao { +public interface RuleChainRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + "AND LOWER(rc.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 4e2596485b..e18f73a065 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -72,17 +72,19 @@ CREATE TABLE IF NOT EXISTS entity_alarm ( CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS asset ( - id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - type varchar(255), - CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) +CREATE TABLE IF NOT EXISTS asset( + id uuid NOT NULL + CONSTRAINT asset_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + type varchar(255), + external_id uuid, + CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); CREATE TABLE IF NOT EXISTS audit_log ( @@ -128,47 +130,53 @@ CREATE TABLE IF NOT EXISTS component_descriptor ( ); CREATE TABLE IF NOT EXISTS customer ( - id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - search_text varchar(255), - state varchar(255), - tenant_id uuid, - title varchar(255), - zip varchar(255) -); - -CREATE TABLE IF NOT EXISTS dashboard ( - id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, - created_time bigint NOT NULL, - configuration varchar, + id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + search_text varchar(255), + state varchar(255), + tenant_id uuid, + title varchar(255), + zip varchar(255), + external_id uuid +); + +CREATE TABLE IF NOT EXISTS dashboard +( + id uuid NOT NULL + CONSTRAINT dashboard_pkey PRIMARY KEY, + created_time bigint NOT NULL, + configuration varchar, assigned_customers varchar(1000000), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - mobile_hide boolean DEFAULT false, - mobile_order int, - image varchar(1000000) + search_text varchar(255), + tenant_id uuid, + title varchar(255), + mobile_hide boolean DEFAULT false, + mobile_order int, + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( - id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - configuration varchar(10000000), - name varchar(255), - type varchar(255), - first_rule_node_id uuid, - root boolean, - debug_mode boolean, - search_text varchar(255), - tenant_id uuid + id uuid NOT NULL + CONSTRAINT rule_chain_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + configuration varchar(10000000), + name varchar(255), + type varchar(255), + first_rule_node_id uuid, + root boolean, + debug_mode boolean, + search_text varchar(255), + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -216,30 +224,31 @@ CREATE TABLE IF NOT EXISTS ota_package ( ); CREATE TABLE IF NOT EXISTS device_profile ( - id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - type varchar(255), - image varchar(1000000), - transport_type varchar(255), - provision_type varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - tenant_id uuid, - firmware_id uuid, - software_id uuid, - default_rule_chain_id uuid, - default_dashboard_id uuid, - default_queue_name varchar(255), - provision_device_key varchar, - CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), - CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), - CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), - CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) + id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + type varchar(255), + image varchar(1000000), + transport_type varchar(255), + provision_type varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + tenant_id uuid, + firmware_id uuid, + software_id uuid, + default_rule_chain_id uuid, + default_dashboard_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, + external_id uuid, + CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), + CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain (id), + CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard (id), + CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package (id), + CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package (id) ); ALTER TABLE ota_package @@ -257,23 +266,24 @@ ALTER TABLE ota_package -- ); CREATE TABLE IF NOT EXISTS device ( - id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - device_profile_id uuid NOT NULL, - device_data jsonb, - type varchar(255), - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - firmware_id uuid, - software_id uuid, - CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), - CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) + id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + device_profile_id uuid NOT NULL, + device_data jsonb, + type varchar(255), + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + firmware_id uuid, + software_id uuid, + external_id uuid, + CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile (id), + CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package (id), + CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package (id) ); CREATE TABLE IF NOT EXISTS device_credentials ( From 849513541ecb7cab4e09a1bdf3cf29480bd28ea0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 17:35:34 +0200 Subject: [PATCH 022/178] Entities export/import API refactoring: permission checks and entity lifecycle events --- .../server/controller/AssetController.java | 18 +--- .../server/controller/BaseController.java | 62 ++++++++++++++ .../server/controller/CustomerController.java | 10 +-- .../controller/DashboardController.java | 8 +- .../server/controller/DeviceController.java | 22 +---- .../controller/DeviceProfileController.java | 25 +----- .../EntitiesExportImportController.java | 84 ++++++++++++------- .../controller/RuleChainController.java | 18 +--- .../DefaultEntitiesExportImportService.java | 13 ++- .../expimp/EntitiesExportImportService.java | 5 +- .../expimp/imp/EntityImportResult.java | 26 ++++++ .../expimp/imp/EntityImportService.java | 3 +- .../imp/impl/AbstractEntityImportService.java | 3 +- .../expimp/imp/impl/AssetImportService.java | 8 +- .../imp/impl/CustomerImportService.java | 8 +- .../imp/impl/DashboardImportService.java | 8 +- .../expimp/imp/impl/DeviceImportService.java | 10 ++- .../imp/impl/DeviceProfileImportService.java | 8 +- .../imp/impl/RuleChainImportService.java | 8 +- .../common/data/export/EntityExportData.java | 5 +- .../data/export/impl/AssetExportData.java | 5 ++ .../data/export/impl/CustomerExportData.java | 5 ++ .../data/export/impl/DashboardExportData.java | 5 ++ .../data/export/impl/DeviceExportData.java | 5 ++ .../export/impl/DeviceProfileExportData.java | 5 ++ .../data/export/impl/RuleChainExportData.java | 5 ++ 26 files changed, 240 insertions(+), 142 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 7bd9cb065d..94c714e24f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -156,7 +156,7 @@ public class AssetController extends BaseController { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser()); + onEntityUpdatedOrCreated(getCurrentUser(), savedAsset, null, asset.getId() == null); return savedAsset; } catch (Exception e) { @@ -166,20 +166,6 @@ public class AssetController extends BaseController { } } - private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) { - try { - logEntityAction(user, asset.getId(), asset, - asset.getCustomerId(), - updated ? ActionType.UPDATED : ActionType.ADDED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - - if (updated) { - sendEntityNotificationMsg(asset.getTenantId(), asset.getId(), EdgeEventActionType.UPDATED); - } - } - @ApiOperation(value = "Delete asset (deleteAsset)", notes = "Deletes the asset and all the relations (from and to the asset). Referencing non-existing asset Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -681,7 +667,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user); + onEntityUpdatedOrCreated(user, importedAssetInfo.getEntity(), importedAssetInfo.getOldEntity(), !importedAssetInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index f945cc30b7..210a6bea2d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.OtaPackage; @@ -68,6 +69,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -83,6 +85,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rpc.Rpc; @@ -141,6 +144,7 @@ import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -919,4 +923,62 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + boolean notifyEdge = false; + + EntityType entityType = savedEntity.getId().getEntityType(); + switch (entityType) { + case DEVICE: + tbClusterService.onDeviceUpdated((Device) savedEntity, (Device) oldEntity); + break; + case DEVICE_PROFILE: + DeviceProfile deviceProfile = (DeviceProfile) savedEntity; + DeviceProfile oldDeviceProfile = (DeviceProfile) oldEntity; + boolean isFirmwareChanged = false; + boolean isSoftwareChanged = false; + if (!isNewEntity) { + if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { + isFirmwareChanged = true; + } + if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { + isSoftwareChanged = true; + } + } + tbClusterService.onDeviceProfileChange(deviceProfile, null); + tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), + isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); + notifyEdge = true; + break; + case RULE_CHAIN: // FIXME: events for rule chain metadata + RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); + if (RuleChainType.CORE.equals(ruleChainType)) { + tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), + isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + if (RuleChainType.EDGE.equals(ruleChainType)) { + if (!isNewEntity) { + notifyEdge = true; + } + } + break; + case ASSET: + case CUSTOMER: + case DASHBOARD: + if (!isNewEntity) { + notifyEdge = true; + } + break; + default: + throw new UnsupportedOperationException(); + } + + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, + isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (notifyEdge) { + sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index b419121459..27037f36e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -33,7 +33,6 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; @@ -151,14 +150,7 @@ public class CustomerController extends BaseController { checkEntity(customer.getId(), customer, Resource.CUSTOMER); Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - - logEntityAction(savedCustomer.getId(), savedCustomer, - savedCustomer.getId(), - customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); - - if (customer.getId() != null) { - sendEntityNotificationMsg(savedCustomer.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); - } + onEntityUpdatedOrCreated(getCurrentUser(), savedCustomer, null, customer.getId() == null); return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 49c33045a5..4ea1ca83c7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -187,13 +187,7 @@ public class DashboardController extends BaseController { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - logEntityAction(savedDashboard.getId(), savedDashboard, - null, - dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); - - if (dashboard.getId() != null) { - sendEntityNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); - } + onEntityUpdatedOrCreated(getCurrentUser(), savedDashboard, null, dashboard.getId() == null); return savedDashboard; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 85a21e912c..d2ffafa73e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -195,7 +195,7 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser()); + onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, oldDevice, created); return savedDevice; } catch (Exception e) { @@ -224,10 +224,8 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - tbClusterService.onDeviceUpdated(savedDevice, device); - logEntityAction(savedDevice.getId(), savedDevice, - savedDevice.getCustomerId(), - device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, device, device.getId() == null); return savedDevice; } catch (Exception e) { @@ -237,18 +235,6 @@ public class DeviceController extends BaseController { } } - private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) { - tbClusterService.onDeviceUpdated(savedDevice, oldDevice); - - try { - logEntityAction(user, savedDevice.getId(), savedDevice, - savedDevice.getCustomerId(), - updated ? ActionType.UPDATED : ActionType.ADDED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - } - @ApiOperation(value = "Delete device (deleteDevice)", notes = "Deletes the device, it's credentials and all the relations (from and to the device). Referencing non-existing device Id will cause an error." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -1015,7 +1001,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user); + onEntityUpdatedOrCreated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index 25a35e77fa..240e60be2c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -46,7 +46,6 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; -import java.util.Objects; import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_DATA; @@ -207,32 +206,14 @@ public class DeviceProfileController extends BaseController { checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); - boolean isFirmwareChanged = false; - boolean isSoftwareChanged = false; - + DeviceProfile oldDeviceProfile = null; if (!created) { - DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); - if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { - isFirmwareChanged = true; - } - if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { - isSoftwareChanged = true; - } + oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); - tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), savedDeviceProfile.getId(), - created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - - logEntityAction(savedDeviceProfile.getId(), savedDeviceProfile, - null, - created ? ActionType.ADDED : ActionType.UPDATED, null); - - otaPackageStateService.update(savedDeviceProfile, isFirmwareChanged, isSoftwareChanged); + onEntityUpdatedOrCreated(getCurrentUser(), deviceProfile, oldDeviceProfile, created); - sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(), - deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); return savedDeviceProfile; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile, diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a11a9189f4..07a033ab03 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; @@ -23,21 +24,20 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.export.EntitiesExportRequest; -import org.thingsboard.server.common.data.export.EntitiesExportResponse; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import java.util.UUID; @RestController @@ -51,49 +51,69 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@PathVariable EntityType entityType, + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") @PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); try { - return exportImportService.exportEntity(getTenantId(), entityId); + return exportEntity(getCurrentUser(), entityId); } catch (Exception e) { throw handleException(e); } } - @PostMapping("/export/batch") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { - TenantId tenantId = getTenantId(); - - EntitiesExportResponse exportResponse = new EntitiesExportResponse(); - - Map>>> result = new HashMap<>(); - exportRequest.getEntities().forEach((entityType, entityIds) -> { - List>> exportDataForEntityType = new LinkedList<>(); - entityIds.forEach(entityId -> { - EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); - exportDataForEntityType.add(exportData); - }); - result.put(entityType, exportDataForEntityType); - }); - - exportResponse.setExportData(result); - return exportResponse; - } + +// @PostMapping("/export/batch") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { +// TenantId tenantId = getTenantId(); +// +// EntitiesExportResponse exportResponse = new EntitiesExportResponse(); +// +// Map>>> result = new HashMap<>(); +// exportRequest.getEntities().forEach((entityType, entityIds) -> { +// List>> exportDataForEntityType = new LinkedList<>(); +// entityIds.forEach(entityId -> { +// EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); +// exportDataForEntityType.add(exportData); +// }); +// result.put(entityType, exportDataForEntityType); +// }); +// +// exportResponse.setExportData(result); +// return exportResponse; +// } + // TODO: export and import of batches // TODO: api to export and import whole customer, whole tenant @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public , I extends EntityId, D extends EntityExportData> E importEntity(@RequestBody D exportData) throws ThingsboardException { + public & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(@RequestBody D exportData) throws ThingsboardException { try { - return exportImportService.importEntity(getTenantId(), exportData); + return importEntity(getCurrentUser(), exportData); } catch (Exception e) { throw handleException(e); } } -// public void importEntities(@RequestBody ) + + private , I extends EntityId> EntityExportData> exportEntity(SecurityUser user, I entityId) throws ThingsboardException { + checkEntityId(entityId, Operation.READ); + return exportImportService.exportEntity(getTenantId(), entityId); + } + + private & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(SecurityUser user, D exportData) throws ThingsboardException { + E existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { + checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers + } else { + checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); + } + + EntityImportResult importResult = exportImportService.importEntity(getTenantId(), exportData); + onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); + + return importResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 6479b838b2..aba074ac60 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -78,11 +78,9 @@ import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -254,20 +252,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - if (RuleChainType.CORE.equals(savedRuleChain.getType())) { - tbClusterService.broadcastEntityStateChangeEvent(ruleChain.getTenantId(), savedRuleChain.getId(), - created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - - logEntityAction(savedRuleChain.getId(), savedRuleChain, - null, - created ? ActionType.ADDED : ActionType.UPDATED, null); - - if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { - if (!created) { - sendEntityNotificationMsg(savedRuleChain.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); - } - } + onEntityUpdatedOrCreated(getCurrentUser(), savedRuleChain, null, created); return savedRuleChain; } catch (Exception e) { @@ -294,6 +279,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); + tbClusterService.broadcastEntityStateChangeEvent(savedRuleChain.getTenantId(), savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, null, ActionType.ADDED, null); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 9b384b34dd..26d983a694 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -24,7 +24,9 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.impl.AbstractEntityImportService; import java.util.Collection; import java.util.EnumMap; @@ -51,16 +53,21 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } // FIXME: somehow validate export data - // FIXME: validate permissions for create or update - // FIXME: send entity lifecycle event @Override - public , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData) { + public , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData) { EntityType entityType = exportData.getEntityType(); EntityImportService importService = getImportService(entityType); return importService.importEntity(tenantId, exportData); } + @Override + @SuppressWarnings("unchecked") + public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + return (E) importServices.values().stream().filter(entityImportService -> entityImportService instanceof AbstractEntityImportService) + .findFirst().map(entityImportService -> (AbstractEntityImportService) importServices).get() + .findByExternalOrInternalId(tenantId, externalId); // FIXME !!! + } @SuppressWarnings("unchecked") private > EntityExportService getExportService(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 5d25cb9320..75503d2750 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -19,11 +19,14 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); - , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData); + , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); + + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java new file mode 100644 index 0000000000..9868670076 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +@Data +public class EntityImportResult> { + private E savedEntity; + private E oldEntity; +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index 195ace0b66..e0f469e751 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -23,7 +23,8 @@ import org.thingsboard.server.common.data.id.TenantId; public interface EntityImportService, D extends EntityExportData> { - E importEntity(TenantId tenantId, D exportData); + // FIXME: get rid of boilerplate for import result creation and everything else + EntityImportResult importEntity(TenantId tenantId, D exportData); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 3f33ba04f0..81855ec8dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -38,7 +38,6 @@ public abstract class AbstractEntityImportService ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; @@ -50,7 +49,7 @@ public abstract class AbstractEntityImportService, ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { + public final , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 5e563762d6..0e3c6f3004 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class AssetImportService extends AbstractEntityImportService importEntity(TenantId tenantId, AssetExportData exportData) { Asset asset = exportData.getAsset(); Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... @@ -51,7 +52,10 @@ public class AssetImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedAsset); + importResult.setOldEntity(existingAsset); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java index 9510cc0700..5f98a7fb66 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class CustomerImportService extends AbstractEntityImportService importEntity(TenantId tenantId, CustomerExportData exportData) { Customer customer = exportData.getCustomer(); Customer existingCustomer = findByExternalId(tenantId, customer.getId()); @@ -49,7 +50,10 @@ public class CustomerImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedCustomer); + importResult.setOldEntity(existingCustomer); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java index eab90bb294..210e7ca841 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class DashboardImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DashboardExportData exportData) { Dashboard dashboard = exportData.getDashboard(); Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); @@ -51,7 +52,10 @@ public class DashboardImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDashboard); + importResult.setOldEntity(existingDashboard); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java index 052fa3e1b3..c72ed48b2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.expimp.imp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.DeviceExportData; @@ -26,7 +25,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -38,7 +37,7 @@ public class DeviceImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceExportData exportData) { Device device = exportData.getDevice(); Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! // what if exporting and importing back already exported entity ? (save version and then load it back) @@ -70,7 +69,10 @@ public class DeviceImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDevice); + importResult.setOldEntity(existingDevice); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java index 615f62d389..99efefe812 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class DeviceProfileImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceProfileExportData exportData) { DeviceProfile deviceProfile = exportData.getDeviceProfile(); DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); @@ -54,7 +55,10 @@ public class DeviceProfileImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDeviceProfile); + importResult.setOldEntity(existingDeviceProfile); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java index eb72f58a7f..92daa19a00 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -37,7 +38,7 @@ public class RuleChainImportService extends AbstractEntityImportService importEntity(TenantId tenantId, RuleChainExportData exportData) { RuleChain ruleChain = exportData.getRuleChain(); RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); @@ -69,7 +70,10 @@ public class RuleChainImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedRuleChain); + importResult.setOldEntity(existingRuleChain); + return importResult; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index 36d3cb4b0c..4fd1a5c732 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -43,6 +43,9 @@ import org.thingsboard.server.common.data.id.HasId; public interface EntityExportData> { @JsonIgnore - EntityType getEntityType(); // fixme: maybe remove if not needed, as well as generic + E getMainEntity(); + + @JsonIgnore + EntityType getEntityType(); // fixme: maybe remove if not needed } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java index c0f417bd76..abb268eeb6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -25,6 +25,11 @@ public class AssetExportData implements EntityExportData { private Asset asset; + @Override + public Asset getMainEntity() { + return asset; + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java index 31e53db223..02ec9e7efa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -25,6 +25,11 @@ public class CustomerExportData implements EntityExportData { private Customer customer; + @Override + public Customer getMainEntity() { + return customer; + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java index 7860c6c553..f75f2479f6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -25,6 +25,11 @@ public class DashboardExportData implements EntityExportData { private Dashboard dashboard; + @Override + public Dashboard getMainEntity() { + return dashboard; + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java index 05f3299d02..ff3741dabc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -27,6 +27,11 @@ public class DeviceExportData implements EntityExportData { private Device device; private DeviceCredentials credentials; + @Override + public Device getMainEntity() { + return device; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java index c030d0b86e..a5d7872cc6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -25,6 +25,11 @@ public class DeviceProfileExportData implements EntityExportData private DeviceProfile deviceProfile; + @Override + public DeviceProfile getMainEntity() { + return deviceProfile; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java index 11fbace911..101f516881 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -27,6 +27,11 @@ public class RuleChainExportData implements EntityExportData { private RuleChain ruleChain; private RuleChainMetaData metaData; + @Override + public RuleChain getMainEntity() { + return ruleChain; + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; From 99d0197caa4dc0c31d3ac726b88cec33c96d2bff Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 18:01:44 +0200 Subject: [PATCH 023/178] Refactor AbstractEntityImportService --- .../server/controller/BaseController.java | 10 ++--- .../DefaultEntitiesExportImportService.java | 32 +++++++++++----- .../expimp/EntitiesExportImportService.java | 4 +- .../expimp/ExportableEntitiesService.java | 26 +++++++++++++ .../imp/impl/AbstractEntityImportService.java | 38 ++++--------------- 5 files changed, 61 insertions(+), 49 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 210a6bea2d..e244bcf680 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -925,7 +925,7 @@ public abstract class BaseController { } public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { - boolean notifyEdge = false; + boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); switch (entityType) { @@ -949,7 +949,7 @@ public abstract class BaseController { tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); - notifyEdge = true; + notifyEdgeService = true; break; case RULE_CHAIN: // FIXME: events for rule chain metadata RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); @@ -959,7 +959,7 @@ public abstract class BaseController { } if (RuleChainType.EDGE.equals(ruleChainType)) { if (!isNewEntity) { - notifyEdge = true; + notifyEdgeService = true; } } break; @@ -967,7 +967,7 @@ public abstract class BaseController { case CUSTOMER: case DASHBOARD: if (!isNewEntity) { - notifyEdge = true; + notifyEdgeService = true; } break; default: @@ -976,7 +976,7 @@ public abstract class BaseController { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (notifyEdge) { + if (notifyEdgeService) { sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 26d983a694..352d26d177 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -22,23 +22,25 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.impl.AbstractEntityImportService; import java.util.Collection; -import java.util.EnumMap; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; // FIXME: review packages and classes naming @Service @TbCoreComponent public class DefaultEntitiesExportImportService implements EntitiesExportImportService { - private final Map> exportServices = new EnumMap<>(EntityType.class); - private final Map> importServices = new EnumMap<>(EntityType.class); + private final Map> exportServices = new HashMap<>(); + private final Map> importServices = new HashMap<>(); + private final Map> daos = new HashMap<>(); // TODO: export and import of the whole tenant @@ -61,14 +63,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importService.importEntity(tenantId, exportData); } + @Override - @SuppressWarnings("unchecked") public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { - return (E) importServices.values().stream().filter(entityImportService -> entityImportService instanceof AbstractEntityImportService) - .findFirst().map(entityImportService -> (AbstractEntityImportService) importServices).get() - .findByExternalOrInternalId(tenantId, externalId); // FIXME !!! + ExportableEntityDao dao = getDao(externalId.getEntityType()); + return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) + .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); } + @SuppressWarnings("unchecked") private > EntityExportService getExportService(EntityType entityType) { return (EntityExportService) exportServices.get(entityType); @@ -79,15 +82,24 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } + @SuppressWarnings("unchecked") + private ExportableEntityDao getDao(EntityType entityType) { + return (ExportableEntityDao) daos.get(entityType); + } + @Autowired - private void setExportImportServices(Collection> exportServices, - Collection> importServices) { + private void setServices(Collection> exportServices, + Collection> importServices, + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); importServices.forEach(entityImportService -> { this.importServices.put(entityImportService.getEntityType(), entityImportService); }); + daos.forEach(dao -> { + this.daos.put(dao.getEntityType(), dao); + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 75503d2750..2a2ac3e1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -21,12 +21,10 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.imp.EntityImportResult; -public interface EntitiesExportImportService { +public interface EntitiesExportImportService extends ExportableEntitiesService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); - } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java new file mode 100644 index 0000000000..0af93dfeec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntitiesService { + + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 81855ec8dd..795af18db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -16,57 +16,33 @@ package org.thingsboard.server.service.expimp.imp.impl; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.EntityType; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.service.expimp.ExportableEntitiesService; import org.thingsboard.server.service.expimp.imp.EntityImportService; -import java.util.Collection; -import java.util.EnumMap; -import java.util.Map; -import java.util.Optional; - public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { - private final Map> daos = new EnumMap<>(EntityType.class); + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; - public final E findByExternalId(TenantId tenantId, I externalId) { - return findByExternalOrInternalId(tenantId, externalId); + protected final E findByExternalId(TenantId tenantId, I externalId) { + return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); } protected final ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; } - HasId entity = findByExternalOrInternalId(tenantId, externalId); + HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); if (entity == null) { throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } return entity.getId(); } - public final , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { - ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); - return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) - .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); - } - - - @SuppressWarnings("unchecked") - private ExportableEntityDao getDao(EntityType entityType) { - return (ExportableEntityDao) daos.get(entityType); - } - - @Autowired - private void setDaos(Collection> daos) { - daos.forEach(dao -> this.daos.put(dao.getEntityType(), dao)); - if (!this.daos.containsKey(getEntityType())) { - throw new IllegalStateException(getClass().getSimpleName() + " requires ExportableEntityDao for entity type " + getEntityType()); - } - } - } From 01ad35d8645f51b763a1d3acf2c6cc72f8188bd1 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 20:02:51 +0200 Subject: [PATCH 024/178] Get rid of boilerplate in import services --- .../EntitiesExportImportController.java | 14 +++---- .../DefaultEntitiesExportImportService.java | 14 +++---- .../expimp/EntitiesExportImportService.java | 6 +-- .../expimp/ExportableEntitiesService.java | 4 +- .../expimp/exp/EntityExportService.java | 4 +- .../expimp/imp/EntityImportResult.java | 4 +- .../expimp/imp/EntityImportService.java | 5 +-- .../imp/impl/AbstractEntityImportService.java | 40 ++++++++++++++++++- .../expimp/imp/impl/AssetImportService.java | 22 +--------- .../imp/impl/CustomerImportService.java | 22 +--------- .../imp/impl/DashboardImportService.java | 20 ++-------- .../expimp/imp/impl/DeviceImportService.java | 38 ++---------------- .../imp/impl/DeviceProfileImportService.java | 22 +--------- .../imp/impl/RuleChainImportService.java | 27 +++---------- .../server/common/data/Customer.java | 3 +- .../server/common/data/Dashboard.java | 3 +- .../server/common/data/Device.java | 3 +- .../server/common/data/DeviceProfile.java | 3 +- .../server/common/data/asset/Asset.java | 3 +- .../data/export/EntitiesExportResponse.java | 3 +- .../common/data/export/EntityExportData.java | 3 +- .../common/data/export/ExportableEntity.java | 39 ++++++++++++++++++ .../server/common/data/rule/RuleChain.java | 3 +- 23 files changed, 133 insertions(+), 172 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 07a033ab03..a3f26bf7b0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -24,13 +24,11 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; @@ -88,7 +86,7 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(@RequestBody D exportData) throws ThingsboardException { + public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData) throws ThingsboardException { try { return importEntity(getCurrentUser(), exportData); } catch (Exception e) { @@ -97,20 +95,20 @@ public class EntitiesExportImportController extends BaseController { } - private , I extends EntityId> EntityExportData> exportEntity(SecurityUser user, I entityId) throws ThingsboardException { + private EntityExportData> exportEntity(SecurityUser user, EntityId entityId) throws ThingsboardException { checkEntityId(entityId, Operation.READ); return exportImportService.exportEntity(getTenantId(), entityId); } - private & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(SecurityUser user, D exportData) throws ThingsboardException { - E existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData) throws ThingsboardException { + ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); if (existingEntity != null) { checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers } else { checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } - EntityImportResult importResult = exportImportService.importEntity(getTenantId(), exportData); + EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData); onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); return importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 352d26d177..c6fe03a750 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -19,8 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -47,7 +47,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS // TODO: export and import of the whole customer ? // TODO: relations export and import @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { EntityType entityType = entityId.getEntityType(); EntityExportService exportService = getExportService(entityType); @@ -56,16 +56,16 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS // FIXME: somehow validate export data @Override - public , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData) { + public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData) { EntityType entityType = exportData.getEntityType(); - EntityImportService importService = getImportService(entityType); + EntityImportService> importService = getImportService(entityType); return importService.importEntity(tenantId, exportData); } @Override - public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { ExportableEntityDao dao = getDao(externalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); @@ -73,12 +73,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @SuppressWarnings("unchecked") - private > EntityExportService getExportService(EntityType entityType) { + private > EntityExportService getExportService(EntityType entityType) { return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") - private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { return (EntityImportService) importServices.get(entityType); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 2a2ac3e1f5..10361865a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -16,15 +16,15 @@ package org.thingsboard.server.service.expimp; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.imp.EntityImportResult; public interface EntitiesExportImportService extends ExportableEntitiesService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); - , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); + , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java index 0af93dfeec..ca401db64b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.service.expimp; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java index 9afa01ff8e..bdd525d5d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.expimp.exp; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityExportService> { +public interface EntityExportService> { // FIXME: export relations // FIXME: get rid of boilerplate diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java index 9868670076..90f5d42bc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java @@ -16,11 +16,11 @@ package org.thingsboard.server.service.expimp.imp; import lombok.Data; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; @Data -public class EntityImportResult> { +public class EntityImportResult> { private E savedEntity; private E oldEntity; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index e0f469e751..96a0f732ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -17,13 +17,12 @@ package org.thingsboard.server.service.expimp.imp; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityImportService, D extends EntityExportData> { +public interface EntityImportService, D extends EntityExportData> { - // FIXME: get rid of boilerplate for import result creation and everything else EntityImportResult importEntity(TenantId tenantId, D exportData); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 795af18db3..fd635e239c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -18,22 +18,58 @@ package org.thingsboard.server.service.expimp.imp.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.ExportableEntitiesService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; -public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { +public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; + // FIXME what if exporting and importing back already exported entity ? (save version and then load it back) + /* + * export entity -> id from env1 -> import this entity -> ... + * + * maybe find not only by external id but by internal too ? but then what if we will try + * */ + @Override + public final EntityImportResult importEntity(TenantId tenantId, D exportData) { + E entity = exportData.getMainEntity(); + E existingEntity = findByExternalId(tenantId, entity.getId()); - protected final E findByExternalId(TenantId tenantId, I externalId) { + entity.setExternalId(entity.getId()); + entity.setTenantId(tenantId); + + if (existingEntity == null) { + entity.setId(null); + } else { + entity.setId(existingEntity.getId()); + } + + E savedEntity = prepareAndSaveEntity(tenantId, entity, existingEntity, exportData); + + EntityImportResult importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedEntity); + importResult.setOldEntity(existingEntity); + return importResult; + } + + protected abstract E prepareAndSaveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + + + private E findByExternalId(TenantId tenantId, I externalId) { return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); } + // TODO or maybe set as additional config whether to update related entities when device already exists ? + // TODO: or also whether to ignore not found internal ids + + // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants protected final ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 0e3c6f3004..4e6fa2b33e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,27 +34,10 @@ public class AssetImportService extends AbstractEntityImportService importEntity(TenantId tenantId, AssetExportData exportData) { - Asset asset = exportData.getAsset(); - Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... - - asset.setExternalId(asset.getId()); - asset.setTenantId(tenantId); - - if (existingAsset == null) { - asset.setId(null); - } else { - asset.setId(existingAsset.getId()); - } - + protected Asset prepareAndSaveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { asset.setCustomerId(getInternalId(tenantId, asset.getCustomerId())); - Asset savedAsset = assetService.saveAsset(asset); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedAsset); - importResult.setOldEntity(existingAsset); - return importResult; + return assetService.saveAsset(asset); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java index 5f98a7fb66..559cc1cd18 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,25 +34,8 @@ public class CustomerImportService extends AbstractEntityImportService importEntity(TenantId tenantId, CustomerExportData exportData) { - Customer customer = exportData.getCustomer(); - Customer existingCustomer = findByExternalId(tenantId, customer.getId()); - - customer.setExternalId(customer.getId()); - customer.setTenantId(tenantId); - - if (existingCustomer == null) { - customer.setId(null); - } else { - customer.setId(existingCustomer.getId()); - } - - Customer savedCustomer = customerService.saveCustomer(customer); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedCustomer); - importResult.setOldEntity(existingCustomer); - return importResult; + protected Customer prepareAndSaveEntity(TenantId tenantId, Customer customer, Customer existingCustomer, CustomerExportData exportData) { + return customerService.saveCustomer(customer); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java index 210e7ca841..916d95ea1c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,27 +34,14 @@ public class DashboardImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DashboardExportData exportData) { - Dashboard dashboard = exportData.getDashboard(); - Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); - - dashboard.setExternalId(dashboard.getId()); - dashboard.setTenantId(tenantId); - + protected Dashboard prepareAndSaveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { if (existingDashboard == null) { - dashboard.setId(null); dashboard.setAssignedCustomers(null); // FIXME: need to assign dashboard to customers ? } else { - dashboard.setId(existingDashboard.getId()); - dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); // we left them untouched (FIXME) + dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); } - Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDashboard); - importResult.setOldEntity(existingDashboard); - return importResult; + return dashboardService.saveDashboard(dashboard); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java index c72ed48b2f..c41e586b83 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.expimp.imp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.DeviceExportData; @@ -25,7 +24,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,44 +33,14 @@ public class DeviceImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceExportData exportData) { - Device device = exportData.getDevice(); - Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! - // what if exporting and importing back already exported entity ? (save version and then load it back) - /* - * export entity -> id from env1 -> import this entity -> ... - * - * maybe find not only by external id but by internal too ? but then what if we will try - * */ - - device.setExternalId(device.getId()); - device.setTenantId(tenantId); - - if (existingDevice == null) { - device.setId(null); - device.setCustomerId(null); // FIXME: find and set customer - } else { - device.setId(existingDevice.getId()); - device.setCustomerId(existingDevice.getCustomerId()); - } - - // TODO or maybe set as additional config whether to update related entities when device already exists ? - // TODO: or also whether to ignore not found internal ids - - // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants - + protected Device prepareAndSaveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { + device.setCustomerId(getInternalId(tenantId, device.getCustomerId())); device.setDeviceProfileId(getInternalId(tenantId, device.getDeviceProfileId())); device.setFirmwareId(getInternalId(tenantId, device.getFirmwareId())); device.setSoftwareId(getInternalId(tenantId, device.getSoftwareId())); - Device savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDevice); - importResult.setOldEntity(existingDevice); - return importResult; + return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java index 99efefe812..8be8bf6b11 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,30 +34,13 @@ public class DeviceProfileImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceProfileExportData exportData) { - DeviceProfile deviceProfile = exportData.getDeviceProfile(); - DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); - - deviceProfile.setExternalId(deviceProfile.getId()); - deviceProfile.setTenantId(tenantId); - - if (existingDeviceProfile == null) { - deviceProfile.setId(null); - } else { - deviceProfile.setId(existingDeviceProfile.getId()); - } - + protected DeviceProfile prepareAndSaveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { deviceProfile.setDefaultRuleChainId(getInternalId(tenantId, deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(getInternalId(tenantId, deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(getInternalId(tenantId, deviceProfile.getFirmwareId())); deviceProfile.setSoftwareId(getInternalId(tenantId, deviceProfile.getSoftwareId())); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDeviceProfile); - importResult.setOldEntity(existingDeviceProfile); - return importResult; + return deviceProfileService.saveDeviceProfile(deviceProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java index 92daa19a00..dbc6b82545 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -38,27 +37,16 @@ public class RuleChainImportService extends AbstractEntityImportService importEntity(TenantId tenantId, RuleChainExportData exportData) { - RuleChain ruleChain = exportData.getRuleChain(); - RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); - - ruleChain.setExternalId(ruleChain.getId()); - ruleChain.setTenantId(tenantId); + protected RuleChain prepareAndSaveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting - - if (existingRuleChain == null) { - ruleChain.setId(null); - } else { - ruleChain.setId(existingRuleChain.getId()); + if (existingRuleChain != null) { + ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); } - RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChain = ruleChainService.saveRuleChain(ruleChain); - if (ruleChain.getId() != null) { - ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); - } RuleChainMetaData metaData = exportData.getMetaData(); - metaData.setRuleChainId(savedRuleChain.getId()); + metaData.setRuleChainId(ruleChain.getId()); metaData.getNodes().forEach(ruleNode -> { ruleNode.setId(null); ruleNode.setRuleChainId(null); @@ -70,10 +58,7 @@ public class RuleChainImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedRuleChain); - importResult.setOldEntity(existingRuleChain); - return importResult; + return ruleChain; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index 065e0c5db5..f87b44b0eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -22,12 +22,13 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -public class Customer extends ContactBased implements HasTenantId { +public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 4b69534bc0..c8b0df98b4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -19,9 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; -public class Dashboard extends DashboardInfo { +public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 519cffc39a..3bd369591f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -25,6 +25,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -40,7 +41,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) @Slf4j -public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage { +public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 6161cd6a5c..0bd770d02d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -43,7 +44,7 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn @ToString(exclude = {"image", "profileDataBytes"}) @EqualsAndHashCode(callSuper = true) @Slf4j -public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage { +public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 6998485460273302018L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index e0924c63ee..694a0162bc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +36,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) -public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { +public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java index cab75313d4..fdf08b2cdb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java @@ -18,12 +18,11 @@ package org.thingsboard.server.common.data.export; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import java.util.List; import java.util.Map; @Data public class EntitiesExportResponse { - private Map>>> exportData; + private Map>>> exportData; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index 4fd1a5c732..b428c3e6b5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) @@ -40,7 +39,7 @@ import org.thingsboard.server.common.data.id.HasId; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) -public interface EntityExportData> { +public interface EntityExportData> { @JsonIgnore E getMainEntity(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java new file mode 100644 index 0000000000..89b0cc3de7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.springframework.data.annotation.Transient; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntity extends HasId, HasTenantId, HasName { + + @Transient @JsonIgnore + I getId(); + @Transient @JsonIgnore + void setId(I id); + + I getExternalId(); + void setExternalId(I externalId); + + TenantId getTenantId(); + void setTenantId(TenantId tenantId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 7ff2b0dc2b..370087deb3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Data @EqualsAndHashCode(callSuper = true) @Slf4j -public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { +public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, ExportableEntity { private static final long serialVersionUID = -5656679015121935465L; From 5ee056ff8da694659a5ceb98f3cf92209a40126a Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 25 Mar 2022 15:32:54 +0200 Subject: [PATCH 025/178] Export/import API refactoring --- .../EntitiesExportImportController.java | 70 +++++++++---------- .../DefaultEntitiesExportImportService.java | 37 ++++++---- .../expimp/EntitiesExportImportService.java | 6 +- .../expimp/ExportableEntitiesService.java | 2 + .../expimp/exp/EntityExportService.java | 6 +- .../expimp/exp/EntityExportSettings.java | 17 ++--- .../exp/impl/AbstractEntityExportService.java | 63 +++++++++++++++++ .../expimp/exp/impl/AssetExportService.java | 17 +---- .../exp/impl/CustomerExportService.java | 17 +---- .../exp/impl/DashboardExportService.java | 17 +---- .../expimp/exp/impl/DeviceExportService.java | 18 +++-- .../exp/impl/DeviceProfileExportService.java | 17 +---- .../exp/impl/RuleChainExportService.java | 16 ++--- .../expimp/imp/EntityImportService.java | 8 ++- .../expimp/imp/EntityImportSettings.java | 18 ++--- .../imp/impl/AbstractEntityImportService.java | 34 +++++---- .../expimp/imp/impl/AssetImportService.java | 3 +- .../imp/impl/CustomerImportService.java | 3 +- .../imp/impl/DashboardImportService.java | 5 +- .../expimp/imp/impl/DeviceImportService.java | 3 +- .../imp/impl/DeviceProfileImportService.java | 3 +- .../imp/impl/RuleChainImportService.java | 7 +- .../server/common/data/Customer.java | 2 +- .../common/data/export/EntityExportData.java | 13 ++-- .../common/data/export/ExportableEntity.java | 4 -- .../data/export/impl/AssetExportData.java | 11 +-- .../data/export/impl/CustomerExportData.java | 11 +-- .../data/export/impl/DashboardExportData.java | 11 +-- .../data/export/impl/DeviceExportData.java | 10 +-- .../export/impl/DeviceProfileExportData.java | 11 +-- .../data/export/impl/RuleChainExportData.java | 10 +-- .../server/common/data/id/IdBased.java | 37 +++++----- .../server/dao/ExportableEntityDao.java | 2 +- 33 files changed, 265 insertions(+), 244 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java => application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java (65%) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java rename common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java => application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java (61%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a3f26bf7b0..96bc70656f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,11 +17,13 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; +import org.apache.commons.collections.CollectionUtils; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -29,14 +31,18 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities") @@ -49,66 +55,60 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") + @PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportInboundRelations(exportInboundRelations) + .build(); try { - return exportEntity(getCurrentUser(), entityId); + return exportEntity(getCurrentUser(), entityId, exportSettings); } catch (Exception e) { throw handleException(e); } } -// @PostMapping("/export/batch") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { -// TenantId tenantId = getTenantId(); -// -// EntitiesExportResponse exportResponse = new EntitiesExportResponse(); -// -// Map>>> result = new HashMap<>(); -// exportRequest.getEntities().forEach((entityType, entityIds) -> { -// List>> exportDataForEntityType = new LinkedList<>(); -// entityIds.forEach(entityId -> { -// EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); -// exportDataForEntityType.add(exportData); -// }); -// result.put(entityType, exportDataForEntityType); -// }); -// -// exportResponse.setExportData(result); -// return exportResponse; -// } - // TODO: export and import of batches - // TODO: api to export and import whole customer, whole tenant - - @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData) throws ThingsboardException { + public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData, + @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { + EntityImportSettings importSettings = EntityImportSettings.builder() + .importInboundRelations(importInboundRelations) + .build(); try { - return importEntity(getCurrentUser(), exportData); + return importEntity(getCurrentUser(), exportData, importSettings); } catch (Exception e) { throw handleException(e); } } - private EntityExportData> exportEntity(SecurityUser user, EntityId entityId) throws ThingsboardException { + // TODO [viacheslav]: export and import of batches + // TODO [viacheslav]: api to export and import whole customer, whole tenant + + private EntityExportData> exportEntity(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); - return exportImportService.exportEntity(getTenantId(), entityId); + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); } - private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData) throws ThingsboardException { + private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers + if (existingEntity != null) {// todo [viacheslav] maybe need to extract permission check to BaseController and put there permission checks from other controllers + checkEntityId(existingEntity.getId(), Operation.WRITE); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + for (EntityId fromId : exportData.getInboundRelations().stream().map(EntityRelation::getFrom).collect(Collectors.toSet())) { + // FIXME [viacheslav]: fromId is external +// checkEntityId(fromId, Operation.WRITE); + } + } } else { checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } - EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData); + EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData, importSettings); onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); return importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index c6fe03a750..0136bd1d52 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -25,56 +25,63 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -// FIXME: review packages and classes naming +// FIXME [viacheslav]: review packages and classes naming @Service @TbCoreComponent public class DefaultEntitiesExportImportService implements EntitiesExportImportService { - private final Map> exportServices = new HashMap<>(); + private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); - // TODO: export and import of the whole tenant - // TODO: export and import of the whole customer ? - // TODO: relations export and import + // TODO [viacheslav]: export and import of the whole tenant + // TODO [viacheslav]: export and import of the whole customer ? @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { EntityType entityType = entityId.getEntityType(); - EntityExportService exportService = getExportService(entityType); + EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(tenantId, entityId); + return exportService.getExportData(tenantId, entityId, exportSettings); } - // FIXME: somehow validate export data + // FIXME [viacheslav]: somehow validate export data @Override - public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData) { + public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - return importService.importEntity(tenantId, exportData); + return importService.importEntity(tenantId, exportData, importSettings); } + @Override + public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + ExportableEntityDao dao = getDao(id.getEntityType()); + return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); + } + @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { ExportableEntityDao dao = getDao(externalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) - .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); + .orElseGet(() -> findEntityById(tenantId, externalId)); } @SuppressWarnings("unchecked") - private > EntityExportService getExportService(EntityType entityType) { - return (EntityExportService) exportServices.get(entityType); + private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") @@ -88,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } @Autowired - private void setServices(Collection> exportServices, + private void setServices(Collection> exportServices, Collection> importServices, Collection> daos) { exportServices.forEach(entityExportService -> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 10361865a2..752d4b6315 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -19,12 +19,14 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; public interface EntitiesExportImportService extends ExportableEntitiesService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); - , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData); + , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java index ca401db64b..17151e3889 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntitiesService { + , I extends EntityId> E findEntityById(TenantId tenantId, I id); + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java index bdd525d5d9..7c7fc49240 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -21,11 +21,9 @@ import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityExportService> { +public interface EntityExportService, D extends EntityExportData> { - // FIXME: export relations - // FIXME: get rid of boilerplate - EntityExportData getExportData(TenantId tenantId, I entityId); + D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings); EntityType getEntityType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java similarity index 65% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java rename to application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java index fdf08b2cdb..c70b0b2216 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.expimp.exp; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; -import java.util.Map; +import lombok.NoArgsConstructor; @Data -public class EntitiesExportResponse { - private Map>>> exportData; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityExportSettings { + private boolean exportInboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java new file mode 100644 index 0000000000..f6c2f5d5f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.expimp.ExportableEntitiesService; +import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; + +import java.util.List; + +public abstract class AbstractEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + + + @Override + public final D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { + D exportData = newExportData(); + + E mainEntity = exportableEntitiesService.findEntityById(tenantId, entityId); + exportData.setMainEntity(mainEntity); + setRelatedEntities(tenantId, mainEntity, exportData); + if (exportSettings.isExportInboundRelations()) { + exportData.setInboundRelations(getInboundRelations(tenantId, entityId)); + } + + return exportData; + } + + protected List getInboundRelations(TenantId tenantId, I entityId) { + return relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); // FIXME [viacheslav]: RelationTypeGroup + } + + protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} + + protected abstract D newExportData(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java index bb4b68bb29..ec1b2dddc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class AssetExportService implements EntityExportService { - - private final AssetService assetService; - +public class AssetExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, AssetId assetId) { - AssetExportData exportData = new AssetExportData(); - exportData.setAsset(assetService.findAssetById(tenantId, assetId)); - return exportData; + protected AssetExportData newExportData() { + return new AssetExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java index 7430f9ddd3..32e33f757c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class CustomerExportService implements EntityExportService { - - private final CustomerService customerService; - +public class CustomerExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, CustomerId customerId) { - CustomerExportData exportData = new CustomerExportData(); - exportData.setCustomer(customerService.findCustomerById(tenantId, customerId)); - return exportData; + protected CustomerExportData newExportData() { + return new CustomerExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java index 87959429d9..be86c41c9a 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DashboardExportService implements EntityExportService { - - private final DashboardService dashboardService; - +public class DashboardExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, DashboardId dashboardId) { - DashboardExportData exportData = new DashboardExportData(); - exportData.setDashboard(dashboardService.findDashboardById(tenantId, dashboardId)); - return exportData; + protected DashboardExportData newExportData() { + return new DashboardExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java index 860aaa9b1d..c3542dc95b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java @@ -19,30 +19,28 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceExportService implements EntityExportService { +public class DeviceExportService extends AbstractEntityExportService { - private final DeviceService deviceService; private final DeviceCredentialsService deviceCredentialsService; @Override - public EntityExportData getExportData(TenantId tenantId, DeviceId deviceId) { - DeviceExportData exportData = new DeviceExportData(); - exportData.setDevice(deviceService.findDeviceById(tenantId, deviceId)); - exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId)); - return exportData; + protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData) { + exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, device.getId())); + } + + @Override + protected DeviceExportData newExportData() { + return new DeviceExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java index 425dbfd5be..03787ab30e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DeviceProfileExportService implements EntityExportService { - - private final DeviceProfileService deviceProfileService; - +public class DeviceProfileExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, DeviceProfileId deviceProfileId) { - DeviceProfileExportData exportData = new DeviceProfileExportData(); - exportData.setDeviceProfile(deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId)); - return exportData; + protected DeviceProfileExportData newExportData() { + return new DeviceProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java index 7a85281d63..0e2d24e990 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java @@ -18,29 +18,29 @@ package org.thingsboard.server.service.expimp.exp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainExportService implements EntityExportService { +public class RuleChainExportService extends AbstractEntityExportService { private final RuleChainService ruleChainService; @Override - public EntityExportData getExportData(TenantId tenantId, RuleChainId ruleChainId) { - RuleChainExportData exportData = new RuleChainExportData(); - exportData.setRuleChain(ruleChainService.findRuleChainById(tenantId, ruleChainId)); - exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId)); - return exportData; + protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData) { + exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChain.getId())); + } + + @Override + protected RuleChainExportData newExportData() { + return new RuleChainExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index 96a0f732ed..693ff310bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -23,7 +23,13 @@ import org.thingsboard.server.common.data.id.TenantId; public interface EntityImportService, D extends EntityExportData> { - EntityImportResult importEntity(TenantId tenantId, D exportData); + /* + * TODO [viacheslav]: should be as options: + * to update related entities e.g. firmware or device profile if entity already exists + * to delete current relations when importing new + * to ignore when cannot find linked entity by external id + * */ + EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); EntityType getEntityType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java similarity index 61% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java index d3502695c0..1b29139e6b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.expimp.imp; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; -import java.util.Map; +import lombok.NoArgsConstructor; @Data -public class EntitiesExportRequest { - // todo: should be processed in a specific order, e.g. device profiles, then devices, then relations, etc. - private Map> entities; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityImportSettings { + private boolean importInboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index fd635e239c..fd27feeea9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.expimp.imp.impl; +import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; @@ -22,23 +23,22 @@ import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.expimp.ExportableEntitiesService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + - // FIXME what if exporting and importing back already exported entity ? (save version and then load it back) - /* - * export entity -> id from env1 -> import this entity -> ... - * - * maybe find not only by external id but by internal too ? but then what if we will try - * */ @Override - public final EntityImportResult importEntity(TenantId tenantId, D exportData) { + public final EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { E entity = exportData.getMainEntity(); E existingEntity = findByExternalId(tenantId, entity.getId()); @@ -51,7 +51,18 @@ public abstract class AbstractEntityImportService { + relation.setTo(savedEntity.getId()); + relation.setFrom(getInternalId(tenantId, relation.getFrom())); + relationService.saveRelation(tenantId, relation); + }); + } EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -59,17 +70,14 @@ public abstract class AbstractEntityImportService ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 4e6fa2b33e..f228b9be5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class AssetImportService extends AbstractEntityImportService { // ruleChainConnectionInfo.setTargetRuleChainId(); - // TODO: check if this thing is needed for "Other Rule Chain Node" - // TODO: and check import of tenant rule chains + // TODO [viacheslav]: check if this thing is needed for "Other Rule Chain Node" + // TODO [viacheslav]: and check import of tenant rule chains }); ruleChainService.saveRuleChainMetaData(tenantId, metaData); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index f87b44b0eb..c2f23a030f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -40,7 +40,7 @@ public class Customer extends ContactBased implements HasTenantId, E private TenantId tenantId; @Getter @Setter - private CustomerId externalId; // FIXME: add to hashcode, equals, etc + private CustomerId externalId; // FIXME [viacheslav]: add to hashcode, equals, etc public Customer() { super(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index b428c3e6b5..261178d913 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.export.impl.CustomerExportData; @@ -28,6 +29,9 @@ import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; + +import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) @@ -39,12 +43,13 @@ import org.thingsboard.server.common.data.id.EntityId; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) -public interface EntityExportData> { +@Data +public abstract class EntityExportData> { - @JsonIgnore - E getMainEntity(); + private E mainEntity; + private List inboundRelations; @JsonIgnore - EntityType getEntityType(); // fixme: maybe remove if not needed + public abstract EntityType getEntityType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java index 89b0cc3de7..d990652298 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.export; -import com.fasterxml.jackson.annotation.JsonIgnore; -import org.springframework.data.annotation.Transient; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; @@ -25,9 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntity extends HasId, HasTenantId, HasName { - @Transient @JsonIgnore I getId(); - @Transient @JsonIgnore void setId(I id); I getExternalId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java index abb268eeb6..6277a56afe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class AssetExportData implements EntityExportData { - - private Asset asset; - - @Override - public Asset getMainEntity() { - return asset; - } +public class AssetExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java index 02ec9e7efa..3de1dbc1b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class CustomerExportData implements EntityExportData { - - private Customer customer; - - @Override - public Customer getMainEntity() { - return customer; - } +public class CustomerExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java index f75f2479f6..7310ad1f92 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class DashboardExportData implements EntityExportData { - - private Dashboard dashboard; - - @Override - public Dashboard getMainEntity() { - return dashboard; - } +public class DashboardExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java index ff3741dabc..7d4d404ebe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -16,22 +16,18 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.security.DeviceCredentials; +@EqualsAndHashCode(callSuper = true) @Data -public class DeviceExportData implements EntityExportData { +public class DeviceExportData extends EntityExportData { - private Device device; private DeviceCredentials credentials; - @Override - public Device getMainEntity() { - return device; - } - @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java index a5d7872cc6..4ebae37b8e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class DeviceProfileExportData implements EntityExportData { - - private DeviceProfile deviceProfile; - - @Override - public DeviceProfile getMainEntity() { - return deviceProfile; - } +public class DeviceProfileExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java index 101f516881..d52c68b99e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -16,22 +16,18 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +@EqualsAndHashCode(callSuper = true) @Data -public class RuleChainExportData implements EntityExportData { +public class RuleChainExportData extends EntityExportData { - private RuleChain ruleChain; private RuleChainMetaData metaData; - @Override - public RuleChain getMainEntity() { - return ruleChain; - } - @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java index cfe6918a7d..5c99ad8ddb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java @@ -16,33 +16,34 @@ package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; -import java.io.Serializable; import java.util.UUID; public abstract class IdBased implements HasId { protected I id; - - public IdBased() { - super(); - } - - public IdBased(I id) { - super(); - this.id = id; - } - public void setId(I id) { - this.id = id; - } + public IdBased() { + super(); + } - public I getId() { - return id; - } + public IdBased(I id) { + super(); + this.id = id; + } + + @JsonSetter + public void setId(I id) { + this.id = id; + } + + public I getId() { + return id; + } - @JsonIgnore - public UUID getUuidId() { + @JsonIgnore + public UUID getUuidId() { if (id != null) { return id.getId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index c7741ec013..4e0822936b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -24,7 +24,7 @@ public interface ExportableEntityDao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); T findByTenantIdAndId(UUID tenantId, UUID id); - // fixme: get rid of boilerplate ? + // fixme [viacheslav]: get rid of boilerplate ? EntityType getEntityType(); From 040b6a31adb7bae03139b843697b2ae4e5cb0849 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 25 Mar 2022 15:54:07 +0200 Subject: [PATCH 026/178] Rename packages for export/import --- .../EntitiesExportImportController.java | 12 ++++++------ .../DefaultEntitiesExportImportService.java | 16 ++++++++-------- .../EntitiesExportImportService.java | 12 ++++++------ .../ExportableEntitiesService.java | 4 ++-- .../exporting}/EntityExportService.java | 6 +++--- .../exporting}/EntityExportSettings.java | 2 +- .../exporting/data}/AssetExportData.java | 3 +-- .../exporting/data}/CustomerExportData.java | 3 +-- .../exporting/data}/DashboardExportData.java | 3 +-- .../exporting/data}/DeviceExportData.java | 3 +-- .../exporting/data}/DeviceProfileExportData.java | 3 +-- .../exporting/data}/EntityExportData.java | 9 ++------- .../exporting/data}/RuleChainExportData.java | 3 +-- .../impl/AbstractEntityExportService.java | 12 ++++++------ .../exporting}/impl/AssetExportService.java | 4 ++-- .../exporting}/impl/CustomerExportService.java | 4 ++-- .../exporting}/impl/DashboardExportService.java | 4 ++-- .../exporting}/impl/DeviceExportService.java | 4 ++-- .../impl/DeviceProfileExportService.java | 4 ++-- .../exporting}/impl/RuleChainExportService.java | 4 ++-- .../importing}/EntityImportResult.java | 4 ++-- .../importing}/EntityImportService.java | 6 +++--- .../importing}/EntityImportSettings.java | 2 +- .../impl/AbstractEntityImportService.java | 14 +++++++------- .../importing}/impl/AssetImportService.java | 6 +++--- .../importing}/impl/CustomerImportService.java | 6 +++--- .../importing}/impl/DashboardImportService.java | 6 +++--- .../importing}/impl/DeviceImportService.java | 6 +++--- .../impl/DeviceProfileImportService.java | 6 +++--- .../importing}/impl/RuleChainImportService.java | 6 +++--- .../thingsboard/server/common/data/Customer.java | 1 - .../server/common/data/Dashboard.java | 1 - .../thingsboard/server/common/data/Device.java | 1 - .../server/common/data/DeviceProfile.java | 1 - .../data/{export => }/ExportableEntity.java | 4 +--- .../server/common/data/asset/Asset.java | 2 +- .../server/common/data/rule/RuleChain.java | 2 +- 37 files changed, 86 insertions(+), 103 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/DefaultEntitiesExportImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/EntitiesExportImportService.java (72%) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/ExportableEntitiesService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/EntityExportService.java (83%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/EntityExportSettings.java (93%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/AssetExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/CustomerExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DashboardExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DeviceExportData.java (89%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DeviceProfileExportData.java (89%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/EntityExportData.java (79%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/RuleChainExportData.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/AbstractEntityExportService.java (83%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/AssetExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/CustomerExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DashboardExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DeviceProfileExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportResult.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportSettings.java (93%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/AbstractEntityImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/AssetImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/CustomerImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DashboardImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DeviceImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DeviceProfileImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/RuleChainImportService.java (91%) rename common/data/src/main/java/org/thingsboard/server/common/data/{export => }/ExportableEntity.java (86%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 96bc70656f..fc3b0b712b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -26,17 +26,17 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.EntitiesExportImportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 0136bd1d52..4e9c42cffd 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -13,22 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import java.util.Collection; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index 752d4b6315..1eb085d219 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; public interface EntitiesExportImportService extends ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java index 17151e3889..bb8f30c4e1 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java index 7c7fc49240..929092e90e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp; +package org.thingsboard.server.service.exportimport.exporting; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java index c70b0b2216..3755ce801d 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp; +package org.thingsboard.server.service.exportimport.exporting; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java index 6277a56afe..4c4872760b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java index 3de1dbc1b9..1fc820814e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java index 7310ad1f92..6ef45fc531 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java index 7d4d404ebe..5f5d8e8f72 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java index 4ebae37b8e..def6b7edbe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java similarity index 79% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index 261178d913..74116276cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -22,12 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.AssetExportData; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java index d52c68b99e..f92616fead 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java @@ -13,12 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java index f6c2f5d5f7..7b298a7d3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.expimp.ExportableEntitiesService; -import org.thingsboard.server.service.expimp.exp.EntityExportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java index ec1b2dddc3..a48167053b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java index 32e33f757c..518a830a21 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java index be86c41c9a..167b57be75 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java index c3542dc95b..8a8bc26820 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java index 03787ab30e..ce96e3b1fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java index 0e2d24e990..8c173fe2ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java index 90f5d42bc9..cc44670757 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import lombok.Data; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @Data diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java index 693ff310bf..b001290cef 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index 1b29139e6b..d153d1378a 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java index fd27feeea9..075991104c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.expimp.ExportableEntitiesService; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index f228b9be5e..6699219388 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 7a8bfa7c9d..251dd4acbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java index dcfba5f17b..b429ba7111 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 50db77fb2e..01312dfa44 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index 8fe7586a25..ef72d982ee 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 5dfe6a8692..5fc1c8fb8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index c2f23a030f..b7287c55af 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index c8b0df98b4..dce910c4ff 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; public class Dashboard extends DashboardInfo implements ExportableEntity { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 3bd369591f..678c68c115 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -25,7 +25,6 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 0bd770d02d..891367fac4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -24,7 +24,6 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java similarity index 86% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index d990652298..f1e8c3754e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -13,10 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.common.data; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index 694a0162bc..da6d9615a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -21,11 +21,11 @@ import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 370087deb3..74186b49bf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -22,10 +22,10 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; From ab5d6f3c0dcb6ad301fbd26005854632c104bdc0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 30 Mar 2022 13:31:43 +0300 Subject: [PATCH 027/178] Entities export/import API refactoring --- .../EntitiesExportImportController.java | 144 ++++++++++++++---- .../DefaultEntitiesExportImportService.java | 26 ++++ .../EntitiesExportImportService.java | 4 + .../exporting/data/EntityExportData.java | 2 + .../server/common/data/Customer.java | 35 +---- .../server/common/data/Dashboard.java | 27 +--- .../server/dao/ExportableEntityDao.java | 7 +- 7 files changed, 153 insertions(+), 92 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index fc3b0b712b..ec2d4273cc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -27,23 +27,37 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.EntitiesExportImportService; import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @RestController @RequestMapping("/api/entities") @TbCoreComponent @@ -51,7 +65,58 @@ import java.util.stream.Collectors; public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; + private final EntityQueryService entityQueryService; + + + // TODO [viacheslav]: export and import of batches + // TODO [viacheslav]: api to export and import whole customer, whole tenant + + + @PostMapping("/exportByFilter") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, + @RequestParam(defaultValue = "false") boolean exportInboundRelations, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME), EntityDataSortOrder.Direction.DESC)); + EntityDataQuery entityDataQuery = new EntityDataQuery(filter, pageLink, List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME)), + Collections.emptyList(), Collections.emptyList()); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { + // FIXME [viacheslav]: check read permission for relation fromId + return entityQueryService.findEntityDataByQuery(user, entityDataQuery).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> { + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + }) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } + @PostMapping("/exportByQuery") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByQuery(@RequestBody EntityDataQuery query, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); +// FIXME [viacheslav]: check read permission for relation fromId + try { + return entityQueryService.findEntityDataByQuery(user, query).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings)) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -60,11 +125,13 @@ public class EntitiesExportImportController extends BaseController { @PathVariable("entityId") UUID entityUuid, @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(exportInboundRelations) - .build(); - try { - return exportEntity(getCurrentUser(), entityId, exportSettings); + checkEntityId(entityId, Operation.READ); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { // FIXME [viacheslav]: check read permission for relation fromId + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); } catch (Exception e) { throw handleException(e); } @@ -73,45 +140,64 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData, - @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { - EntityImportSettings importSettings = EntityImportSettings.builder() - .importInboundRelations(importInboundRelations) - .build(); + public List>> importEntity(@RequestBody List>> exportDataList, + @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityImportSettings importSettings = toImportSettings(importInboundRelations); + + for (EntityExportData> exportData : exportDataList) { + checkPermissionsForImport(user, exportData, importSettings); + } + try { - return importEntity(getCurrentUser(), exportData, importSettings); + return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); } catch (Exception e) { throw handleException(e); } } - - // TODO [viacheslav]: export and import of batches - // TODO [viacheslav]: api to export and import whole customer, whole tenant - - private EntityExportData> exportEntity(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public void checkPermissionsForExport(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + if (exportSettings.isExportInboundRelations()) { + for (EntityRelation entityRelation : relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON)) { + EntityId fromId = entityRelation.getFrom(); + checkEntityId(fromId, Operation.READ); + } + } } - private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { - ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) {// todo [viacheslav] maybe need to extract permission check to BaseController and put there permission checks from other controllers + public void checkPermissionsForImport(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { + ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { checkEntityId(existingEntity.getId(), Operation.WRITE); - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - for (EntityId fromId : exportData.getInboundRelations().stream().map(EntityRelation::getFrom).collect(Collectors.toSet())) { - // FIXME [viacheslav]: fromId is external -// checkEntityId(fromId, Operation.WRITE); + } else { + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + } + + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + for (EntityRelation entityRelation : exportData.getInboundRelations()) { + ExportableEntity entityFrom = exportImportService.findEntityByExternalId(user.getTenantId(), entityRelation.getFrom()); + if (entityFrom != null) { + accessControlService.checkPermission(user, Resource.of(entityFrom.getId().getEntityType()), Operation.WRITE, entityFrom.getId(), entityFrom); + } else { + throw new ThingsboardException("Relation with non-existing entity", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } } - } else { - checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } + } + - EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData, importSettings); - onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); + private EntityImportSettings toImportSettings(boolean importInboundRelations) { + EntityImportSettings importSettings = EntityImportSettings.builder() + .importInboundRelations(importInboundRelations) + .build(); + return importSettings; + } - return importResult; + private EntityExportSettings toExportSettings(boolean exportInboundRelations) { + return EntityExportSettings.builder() + .exportInboundRelations(exportInboundRelations) + .build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 4e9c42cffd..93fa370c94 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,9 +32,12 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportService import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; // FIXME [viacheslav]: review packages and classes naming @Service @@ -44,6 +48,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); + protected static final List SUPPORTED_ENTITY_TYPES = List.of( + EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, + EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD + ); + // TODO [viacheslav]: export and import of the whole tenant // TODO [viacheslav]: export and import of the whole customer ? @@ -55,7 +64,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return exportService.getExportData(tenantId, entityId, exportSettings); } + // FIXME [viacheslav]: somehow validate export data + @Transactional @Override public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { EntityType entityType = exportData.getEntityType(); @@ -64,6 +75,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importService.importEntity(tenantId, exportData, importSettings); } + @Transactional + @Override + public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { + return exportDataList.stream() + .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) + .map(exportData -> importEntity(tenantId, exportData, importSettings)) + .collect(Collectors.toList()); + } + @Override public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { @@ -81,11 +101,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); + } return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); + } return (EntityImportService) importServices.get(entityType); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index 1eb085d219..e8d4a40efe 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -23,10 +23,14 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.List; + public interface EntitiesExportImportService extends ExportableEntitiesService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); + , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings); + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index 74116276cb..f77a5fbb29 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -42,6 +43,7 @@ import java.util.List; public abstract class EntityExportData> { private E mainEntity; + @JsonInclude(JsonInclude.Include.NON_NULL) private List inboundRelations; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index b7287c55af..92e5e9ae78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +@EqualsAndHashCode(callSuper = false) public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; @@ -39,7 +41,7 @@ public class Customer extends ContactBased implements HasTenantId, E private TenantId tenantId; @Getter @Setter - private CustomerId externalId; // FIXME [viacheslav]: add to hashcode, equals, etc + private CustomerId externalId; public Customer() { super(); @@ -167,37 +169,6 @@ public class Customer extends ContactBased implements HasTenantId, E return getTitle(); } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); - result = prime * result + ((title == null) ? 0 : title.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Customer other = (Customer) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; - } - @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index dce910c4ff..59c122f3d0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,10 +17,12 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; +@EqualsAndHashCode(callSuper = false) public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; @@ -60,31 +62,6 @@ public class Dashboard extends DashboardInfo implements ExportableEntity { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); T findByTenantIdAndId(UUID tenantId, UUID id); - // fixme [viacheslav]: get rid of boilerplate ? - EntityType getEntityType(); - /* - * default > ExportableEntityRepository getExportableEntityRepository() { - * ((ExportableEntityRepository) getJpaRepository).find... - * */ + EntityType getEntityType(); } From 6dc962882e954e00cec3caf8679af88461e29d94 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 30 Mar 2022 16:42:33 +0300 Subject: [PATCH 028/178] Add more options for entity export and import settings; refactoring --- .../upgrade/{3.4 => 3.3.4}/schema_update.sql | 0 .../server/controller/BaseController.java | 2 +- .../EntitiesExportImportController.java | 49 ++--- .../exporting/EntityExportSettings.java | 1 + .../exporting/data/EntityExportData.java | 3 +- .../impl/AbstractEntityExportService.java | 12 +- .../importing/EntityImportService.java | 6 - .../importing/EntityImportSettings.java | 4 + .../impl/AbstractEntityImportService.java | 83 ++++++-- .../importing/impl/AssetImportService.java | 9 +- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 17 +- .../importing/impl/DeviceImportService.java | 15 +- .../impl/DeviceProfileImportService.java | 15 +- .../impl/RuleChainImportService.java | 8 +- .../main/resources/sql/schema-entities.sql | 194 +++++++++--------- 16 files changed, 234 insertions(+), 188 deletions(-) rename application/src/main/data/upgrade/{3.4 => 3.3.4}/schema_update.sql (100%) diff --git a/application/src/main/data/upgrade/3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql similarity index 100% rename from application/src/main/data/upgrade/3.4/schema_update.sql rename to application/src/main/data/upgrade/3.3.4/schema_update.sql diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index e244bcf680..7416444104 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index ec2d4273cc..f7513eb2d5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -72,9 +72,28 @@ public class EntitiesExportImportController extends BaseController { // TODO [viacheslav]: api to export and import whole customer, whole tenant + @PostMapping("/export/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") + @PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + checkEntityId(entityId, Operation.READ); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { // FIXME [viacheslav]: check read permission for relation fromId + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + } catch (Exception e) { + throw handleException(e); + } + } + @PostMapping("/exportByFilter") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, + public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, // TODO [viacheslav]: exportInboundRelations, exportOutboundRelations @RequestParam(defaultValue = "false") boolean exportInboundRelations, @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { @@ -118,25 +137,6 @@ public class EntitiesExportImportController extends BaseController { } } - @PostMapping("/export/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") - @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - checkEntityId(entityId, Operation.READ); - - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); - - try { // FIXME [viacheslav]: check read permission for relation fromId - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); - } catch (Exception e) { - throw handleException(e); - } - } - @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -150,7 +150,11 @@ public class EntitiesExportImportController extends BaseController { } try { - return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + List>> importResultList = exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + importResultList.forEach(entityImportResult -> { + onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); + }); + return importResultList; } catch (Exception e) { throw handleException(e); } @@ -188,10 +192,9 @@ public class EntitiesExportImportController extends BaseController { private EntityImportSettings toImportSettings(boolean importInboundRelations) { - EntityImportSettings importSettings = EntityImportSettings.builder() + return EntityImportSettings.builder() .importInboundRelations(importInboundRelations) .build(); - return importSettings; } private EntityExportSettings toExportSettings(boolean exportInboundRelations) { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java index 3755ce801d..f4eea9913a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportInboundRelations; + private boolean exportOutboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index f77a5fbb29..b9cc8c71a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -39,12 +39,13 @@ import java.util.List; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) +@JsonInclude(JsonInclude.Include.NON_NULL) @Data public abstract class EntityExportData> { private E mainEntity; - @JsonInclude(JsonInclude.Include.NON_NULL) private List inboundRelations; + private List outboundRelations; @JsonIgnore public abstract EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java index 7b298a7d3c..ee6437c7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java @@ -45,17 +45,19 @@ public abstract class AbstractEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); + exportData.setInboundRelations(inboundRelations); + } + if (exportSettings.isExportOutboundRelations()) { + List outboundRelations = relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON); + exportData.setOutboundRelations(outboundRelations); } return exportData; } - protected List getInboundRelations(TenantId tenantId, I entityId) { - return relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); // FIXME [viacheslav]: RelationTypeGroup - } - protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} protected abstract D newExportData(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java index b001290cef..8697b04f59 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java @@ -23,12 +23,6 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa public interface EntityImportService, D extends EntityExportData> { - /* - * TODO [viacheslav]: should be as options: - * to update related entities e.g. firmware or device profile if entity already exists - * to delete current relations when importing new - * to ignore when cannot find linked entity by external id - * */ EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index d153d1378a..22b67bcbb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -26,4 +26,8 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean importInboundRelations; + private boolean importOutboundRelations; + private boolean removeExistingRelationsAndSaveNew; + + private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java index 075991104c..b521074bab 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java @@ -18,10 +18,13 @@ package org.thingsboard.server.service.exportimport.importing.impl; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.exportimport.ExportableEntitiesService; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; @@ -29,6 +32,11 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportService; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -36,11 +44,11 @@ public abstract class AbstractEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { + public EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { E entity = exportData.getMainEntity(); - E existingEntity = findByExternalId(tenantId, entity.getId()); + E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); entity.setExternalId(entity.getId()); entity.setTenantId(tenantId); @@ -51,18 +59,19 @@ public abstract class AbstractEntityImportService() { + @Override + public ID get(TenantId tenantId, Function idExtractor) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { + return getInternalId(tenantId, idExtractor.apply(entity)); + } else { + return idExtractor.apply(existingEntity); + } } - exportData.getInboundRelations().forEach(relation -> { - relation.setTo(savedEntity.getId()); - relation.setFrom(getInternalId(tenantId, relation.getFrom())); - relationService.saveRelation(tenantId, relation); - }); - } + }); + + E savedEntity = saveEntity(tenantId, entity, existingEntity, exportData); + importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -70,16 +79,46 @@ public abstract class AbstractEntityImportService idProvider) {} + + protected abstract E saveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + + private void importRelations(TenantId tenantId, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) { + List newRelations = new LinkedList<>(); - private E findByExternalId(TenantId tenantId, I externalId) { - return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + newRelations.addAll(exportData.getInboundRelations().stream() + .peek(relation -> { + relation.setTo(savedEntity.getId()); + relation.setFrom(getInternalId(tenantId, relation.getFrom())); + }) + .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelationsAndSaveNew() && existingEntity != null) { + relationService.findByTo(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { + relationService.deleteRelation(tenantId, existingRelation); + }); + } + } + if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { + newRelations.addAll(exportData.getOutboundRelations().stream() + .peek(relation -> { + relation.setTo(getInternalId(tenantId, relation.getTo())); + relation.setFrom(savedEntity.getId()); + }) + .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelationsAndSaveNew() && existingEntity != null) { + relationService.findByFrom(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { + relationService.deleteRelation(tenantId, existingRelation); + }); + } + } + + newRelations.forEach(relation -> relationService.saveRelation(tenantId, relation)); } - // FIXME [viacheslav]: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants - protected final ID getInternalId(TenantId tenantId, ID externalId) { - if (externalId == null) { + private ID getInternalId(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) { return null; } HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); @@ -89,4 +128,8 @@ public abstract class AbstractEntityImportService { + I get(TenantId tenantId, Function idExtractor); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index 6699219388..0d6e6d01b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,11 +32,13 @@ public class AssetImportService extends AbstractEntityImportService idProvider) { + asset.setCustomerId(idProvider.get(tenantId, Asset::getCustomerId)); + } + @Override + protected Asset saveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 251dd4acbf..22cdc10204 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,9 +32,8 @@ public class CustomerImportService extends AbstractEntityImportService idProvider) { +// if (existingDashboard == null) { +// dashboard.setAssignedCustomers(null); // FIXME [viacheslav]: need to assign dashboard to customers ? +// } else { +// dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); +// } + } + @Override + protected Dashboard saveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { return dashboardService.saveDashboard(dashboard); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 01312dfa44..8c02edec05 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,14 +32,16 @@ public class DeviceImportService extends AbstractEntityImportService idProvider) { + device.setCustomerId(idProvider.get(tenantId, Device::getCustomerId)); + device.setDeviceProfileId(idProvider.get(tenantId, Device::getDeviceProfileId)); + device.setFirmwareId(idProvider.get(tenantId, Device::getFirmwareId)); + device.setSoftwareId(idProvider.get(tenantId, Device::getSoftwareId)); + } + @Override + protected Device saveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index ef72d982ee..9981607481 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,14 +32,16 @@ public class DeviceProfileImportService extends AbstractEntityImportService idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.get(tenantId, DeviceProfile::getDefaultRuleChainId)); + deviceProfile.setDefaultDashboardId(idProvider.get(tenantId, DeviceProfile::getDefaultDashboardId)); + deviceProfile.setFirmwareId(idProvider.get(tenantId, DeviceProfile::getFirmwareId)); + deviceProfile.setSoftwareId(idProvider.get(tenantId, DeviceProfile::getSoftwareId)); + } + @Override + protected DeviceProfile saveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 5fc1c8fb8c..6be2fc4317 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -26,7 +25,6 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -35,10 +33,12 @@ public class RuleChainImportService extends AbstractEntityImportService idProvider) { + } - @Transactional @Override - protected RuleChain prepareAndSaveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData, EntityImportSettings importSettings) { + protected RuleChain saveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting if (existingRuleChain != null) { ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index e18f73a065..9862ded617 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -72,19 +72,18 @@ CREATE TABLE IF NOT EXISTS entity_alarm ( CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS asset( - id uuid NOT NULL - CONSTRAINT asset_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - type varchar(255), - external_id uuid, - CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) +CREATE TABLE IF NOT EXISTS asset ( + id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + type varchar(255), + external_id uuid, + CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); CREATE TABLE IF NOT EXISTS audit_log ( @@ -130,53 +129,50 @@ CREATE TABLE IF NOT EXISTS component_descriptor ( ); CREATE TABLE IF NOT EXISTS customer ( - id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - search_text varchar(255), - state varchar(255), - tenant_id uuid, - title varchar(255), - zip varchar(255), - external_id uuid -); - -CREATE TABLE IF NOT EXISTS dashboard -( - id uuid NOT NULL - CONSTRAINT dashboard_pkey PRIMARY KEY, - created_time bigint NOT NULL, - configuration varchar, + id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + search_text varchar(255), + state varchar(255), + tenant_id uuid, + title varchar(255), + zip varchar(255), + external_id uuid +); + +CREATE TABLE IF NOT EXISTS dashboard ( + id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, + created_time bigint NOT NULL, + configuration varchar, assigned_customers varchar(1000000), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - mobile_hide boolean DEFAULT false, - mobile_order int, - image varchar(1000000), - external_id uuid + search_text varchar(255), + tenant_id uuid, + title varchar(255), + mobile_hide boolean DEFAULT false, + mobile_order int, + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( - id uuid NOT NULL - CONSTRAINT rule_chain_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - configuration varchar(10000000), - name varchar(255), - type varchar(255), - first_rule_node_id uuid, - root boolean, - debug_mode boolean, - search_text varchar(255), - tenant_id uuid, - external_id uuid + id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + configuration varchar(10000000), + name varchar(255), + type varchar(255), + first_rule_node_id uuid, + root boolean, + debug_mode boolean, + search_text varchar(255), + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -224,31 +220,31 @@ CREATE TABLE IF NOT EXISTS ota_package ( ); CREATE TABLE IF NOT EXISTS device_profile ( - id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - type varchar(255), - image varchar(1000000), - transport_type varchar(255), - provision_type varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - tenant_id uuid, - firmware_id uuid, - software_id uuid, - default_rule_chain_id uuid, - default_dashboard_id uuid, - default_queue_name varchar(255), - provision_device_key varchar, - external_id uuid, - CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), - CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain (id), - CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard (id), - CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package (id), - CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package (id) + id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + type varchar(255), + image varchar(1000000), + transport_type varchar(255), + provision_type varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + tenant_id uuid, + firmware_id uuid, + software_id uuid, + default_rule_chain_id uuid, + default_dashboard_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, + external_id uuid, + CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), + CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), + CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), + CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) ); ALTER TABLE ota_package @@ -266,24 +262,24 @@ ALTER TABLE ota_package -- ); CREATE TABLE IF NOT EXISTS device ( - id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - device_profile_id uuid NOT NULL, - device_data jsonb, - type varchar(255), - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - firmware_id uuid, - software_id uuid, - external_id uuid, - CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile (id), - CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package (id), - CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package (id) + id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + device_profile_id uuid NOT NULL, + device_data jsonb, + type varchar(255), + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + firmware_id uuid, + software_id uuid, + external_id uuid, + CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), + CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) ); CREATE TABLE IF NOT EXISTS device_credentials ( From 19a27f93f456266895af6026cc86f5c7d27971a6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 31 Mar 2022 19:41:25 +0300 Subject: [PATCH 029/178] Export/import API improvements and refactoring --- .../EntitiesExportImportController.java | 275 ++++++++++++------ .../DefaultEntitiesExportImportService.java | 8 +- .../EntitiesExportImportService.java | 2 +- .../exporting/impl/AssetExportService.java | 2 +- ...vice.java => BaseEntityExportService.java} | 3 +- .../exporting/impl/CustomerExportService.java | 2 +- .../impl/DashboardExportService.java | 2 +- .../exporting/impl/DeviceExportService.java | 3 +- .../impl/DeviceProfileExportService.java | 2 +- .../impl/RuleChainExportService.java | 3 +- .../importing/EntityImportSettings.java | 3 +- .../importing/impl/AssetImportService.java | 8 +- ...vice.java => BaseEntityImportService.java} | 65 +++-- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 46 ++- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 8 +- .../impl/RuleChainImportService.java | 57 ++-- .../query/DefaultEntityQueryRepository.java | 2 + .../dao/sql/query/EntityKeyMapping.java | 1 + 20 files changed, 320 insertions(+), 184 deletions(-) rename application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/{AbstractEntityExportService.java => BaseEntityExportService.java} (93%) rename application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/{AbstractEntityImportService.java => BaseEntityImportService.java} (68%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f7513eb2d5..411a5ec00e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -27,10 +25,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -38,169 +37,263 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; -import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @RestController @RequestMapping("/api/entities") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") @TbCoreComponent @RequiredArgsConstructor public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final EntityQueryService entityQueryService; + private final ExportableEntitiesService exportableEntitiesService; + private final EntityService entityService; - // TODO [viacheslav]: export and import of batches - // TODO [viacheslav]: api to export and import whole customer, whole tenant - + @PostMapping("/export/{entityType}/{id}") + public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, + @PathVariable UUID id, + @RequestParam Map exportSettingsParams) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); + try { + return exportEntity(getTenantId(), entityId, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } - @PostMapping("/export/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") - @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - checkEntityId(entityId, Operation.READ); + @PostMapping(value = "/export/{entityType}", params = "ids") + public List>> exportEntitiesByIds(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam Map exportSettingsParams) throws ThingsboardException { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + try { + return exportEntitiesByIds(getTenantId(), entitiesIds, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + @PostMapping(value = "/export/{entityType}", params = "ids") + public List>> exportAllEntitiesByEntityType(@PathVariable EntityType entityType, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); - try { // FIXME [viacheslav]: check read permission for relation fromId - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + try { + return exportEntitiesByFilter(tenantId, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } @PostMapping("/exportByFilter") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, // TODO [viacheslav]: exportInboundRelations, exportOutboundRelations - @RequestParam(defaultValue = "false") boolean exportInboundRelations, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME), EntityDataSortOrder.Direction.DESC)); - EntityDataQuery entityDataQuery = new EntityDataQuery(filter, pageLink, List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME)), - Collections.emptyList(), Collections.emptyList()); - - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + public List>> exportEntitiesByFilter(@RequestBody EntityFilter filter, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); + try { + return exportEntitiesByFilter(tenantId, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } + // FIXME: too aggressive + @PostMapping("/exportByFilters") + public List>> exportAllEntitiesByFilters(@RequestBody List filters, + @RequestParam Map exportSettingsParams, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); try { - // FIXME [viacheslav]: check read permission for relation fromId - return entityQueryService.findEntityDataByQuery(user, entityDataQuery).getData().stream() - .map(EntityData::getEntityId) - .map(entityId -> { - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); - }) - .collect(Collectors.toList()); + List>> exportDataList = new ArrayList<>(); + for (EntityFilter filter : filters) { + exportDataList.addAll(exportEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + } + return exportDataList; } catch (Exception e) { throw handleException(e); } } @PostMapping("/exportByQuery") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByQuery(@RequestBody EntityDataQuery query, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); -// FIXME [viacheslav]: check read permission for relation fromId + public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, + @RequestParam Map exportSettingsParams, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); try { - return entityQueryService.findEntityDataByQuery(user, query).getData().stream() - .map(EntityData::getEntityId) - .map(entityId -> exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings)) - .collect(Collectors.toList()); + return exportEntitiesByQuery(tenantId, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } - @PostMapping("/import") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> importEntity(@RequestBody List>> exportDataList, - @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - EntityImportSettings importSettings = toImportSettings(importInboundRelations); + private List>> exportEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - for (EntityExportData> exportData : exportDataList) { - checkPermissionsForImport(user, exportData, importSettings); - } + EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + return exportEntitiesByQuery(tenantId, customerId, query, exportSettings); + } - try { - List>> importResultList = exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); - importResultList.forEach(entityImportResult -> { - onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); - }); - return importResultList; - } catch (Exception e) { - throw handleException(e); + private List>> exportEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { + List entitiesIds = entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + return exportEntitiesByIds(tenantId, entitiesIds, exportSettings); + } + + private List>> exportEntitiesByIds(TenantId tenantId, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { + List>> exportDataList = new ArrayList<>(); + for (EntityId entityId : entitiesIds) { + exportDataList.add(exportEntity(tenantId, entityId, exportSettings)); } + return exportDataList; } - public void checkPermissionsForExport(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { + private , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); + + List relations = new LinkedList<>(); if (exportSettings.isExportInboundRelations()) { - for (EntityRelation entityRelation : relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON)) { - EntityId fromId = entityRelation.getFrom(); - checkEntityId(fromId, Operation.READ); + relations.addAll(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)); + } + if (exportSettings.isExportOutboundRelations()) { + relations.addAll(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)); + } + for (EntityRelation relation : relations) { + if (!relation.getFrom().equals(entityId)) { + checkEntityId(relation.getFrom(), Operation.READ); + } else if (!relation.getTo().equals(entityId)) { + checkEntityId(relation.getTo(), Operation.READ); } } + + return exportImportService.exportEntity(tenantId, entityId, exportSettings); } - public void checkPermissionsForImport(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { - ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); - } else { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + + @PostMapping("/import") + public List>> importEntity(@RequestBody List>> exportDataList, + @RequestParam Map importSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityImportSettings importSettings = toImportSettings(importSettingsParams); + + try { + return importEntities(user, exportDataList, importSettings) + .stream().peek(entityImportResult -> { + onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); + }) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); } + } + - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - for (EntityRelation entityRelation : exportData.getInboundRelations()) { - ExportableEntity entityFrom = exportImportService.findEntityByExternalId(user.getTenantId(), entityRelation.getFrom()); - if (entityFrom != null) { - accessControlService.checkPermission(user, Resource.of(entityFrom.getId().getEntityType()), Operation.WRITE, entityFrom.getId(), entityFrom); - } else { - throw new ThingsboardException("Relation with non-existing entity", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + for (EntityExportData> exportData : exportDataList) { + ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { + checkEntityId(existingEntity.getId(), Operation.WRITE); + } else { + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + } + + List relations = new LinkedList<>(); + if (importSettings.isImportInboundRelations() && exportData.getInboundRelations() != null) { + relations.addAll(exportData.getInboundRelations()); + } + if (importSettings.isImportOutboundRelations() && exportData.getOutboundRelations() != null) { + relations.addAll(exportData.getOutboundRelations()); + } + for (EntityRelation relation : relations) { + EntityId otherEntityId = null; + if (!relation.getFrom().equals(exportData.getMainEntity().getId())) { + otherEntityId = relation.getFrom(); + } else if (!relation.getTo().equals(exportData.getMainEntity().getId())) { + otherEntityId = relation.getTo(); + } + if (otherEntityId != null) { + ExportableEntity otherEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), otherEntityId); + if (otherEntity != null) { + checkEntityId(otherEntity.getId(), Operation.WRITE); + } else { + throw new IllegalArgumentException("Relation is referencing non-existing entity"); + } } } } + + return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); } - private EntityImportSettings toImportSettings(boolean importInboundRelations) { - return EntityImportSettings.builder() - .importInboundRelations(importInboundRelations) + private EntityExportSettings toExportSettings(Map exportSettingsParams) { + return EntityExportSettings.builder() + .exportInboundRelations(getParam(exportSettingsParams, "exportInboundRelations", false, Boolean::parseBoolean)) + .exportOutboundRelations(getParam(exportSettingsParams, "exportOutboundRelations", false, Boolean::parseBoolean)) .build(); } - private EntityExportSettings toExportSettings(boolean exportInboundRelations) { - return EntityExportSettings.builder() - .exportInboundRelations(exportInboundRelations) + private EntityImportSettings toImportSettings(Map importSettingsParams) { + return EntityImportSettings.builder() + .importInboundRelations(getParam(importSettingsParams, "importInboundRelations", false, Boolean::parseBoolean)) + .importOutboundRelations(getParam(importSettingsParams, "importOutboundRelations", false, Boolean::parseBoolean)) + .removeExistingRelations(getParam(importSettingsParams, "removeExistingRelations", true, Boolean::parseBoolean)) + .updateReferencesToOtherEntities(getParam(importSettingsParams, "updateReferencesToOtherEntities", true, Boolean::parseBoolean)) .build(); } + protected T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { + return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); + } + + private CustomerId toCustomerId(UUID customerUuid) { + return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 93fa370c94..8d85355849 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -39,10 +39,9 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -// FIXME [viacheslav]: review packages and classes naming @Service @TbCoreComponent -public class DefaultEntitiesExportImportService implements EntitiesExportImportService { +public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); @@ -54,8 +53,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS ); - // TODO [viacheslav]: export and import of the whole tenant - // TODO [viacheslav]: export and import of the whole customer ? @Override public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { EntityType entityType = entityId.getEntityType(); @@ -65,7 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - // FIXME [viacheslav]: somehow validate export data + // TODO [viacheslav]: validate export data @Transactional @Override public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { @@ -80,6 +77,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { return exportDataList.stream() .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) + // TODO [viacheslav]: order for rule chains (depending on references) .map(exportData -> importEntity(tenantId, exportData, importSettings)) .collect(Collectors.toList()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index e8d4a40efe..682db6ba21 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportSetting import java.util.List; -public interface EntitiesExportImportService extends ExportableEntitiesService { +public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java index a48167053b..c22b345299 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.AssetExportDat @Service @TbCoreComponent -public class AssetExportService extends AbstractEntityExportService { +public class AssetExportService extends BaseEntityExportService { @Override protected AssetExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java index ee6437c7dc..68c78e87c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java @@ -30,14 +30,13 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa import java.util.List; -public abstract class AbstractEntityExportService, D extends EntityExportData> implements EntityExportService { +public abstract class BaseEntityExportService, D extends EntityExportData> implements EntityExportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; - @Override public final D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { D exportData = newExportData(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java index 518a830a21..1f23e8e7d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.CustomerExport @Service @TbCoreComponent -public class CustomerExportService extends AbstractEntityExportService { +public class CustomerExportService extends BaseEntityExportService { @Override protected CustomerExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java index 167b57be75..17d593157b 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.DashboardExpor @Service @TbCoreComponent -public class DashboardExportService extends AbstractEntityExportService { +public class DashboardExportService extends BaseEntityExportService { @Override protected DashboardExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java index 8a8bc26820..b8c7e65b34 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java @@ -28,11 +28,10 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportDa @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceExportService extends AbstractEntityExportService { +public class DeviceExportService extends BaseEntityExportService { private final DeviceCredentialsService deviceCredentialsService; - @Override protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData) { exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, device.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java index ce96e3b1fe..24b5f8891e 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileE @Service @TbCoreComponent -public class DeviceProfileExportService extends AbstractEntityExportService { +public class DeviceProfileExportService extends BaseEntityExportService { @Override protected DeviceProfileExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java index 8c173fe2ce..f1b9a27f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java @@ -28,11 +28,10 @@ import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExpor @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainExportService extends AbstractEntityExportService { +public class RuleChainExportService extends BaseEntityExportService { private final RuleChainService ruleChainService; - @Override protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData) { exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChain.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index 22b67bcbb4..c3cf0b531d 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -27,7 +27,6 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean importInboundRelations; private boolean importOutboundRelations; - private boolean removeExistingRelationsAndSaveNew; - + private boolean removeExistingRelations; private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index 0d6e6d01b5..04e5a5eeca 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -28,17 +28,13 @@ import org.thingsboard.server.service.exportimport.exporting.data.AssetExportDat @Service @TbCoreComponent @RequiredArgsConstructor -public class AssetImportService extends AbstractEntityImportService { +public class AssetImportService extends BaseEntityImportService { private final AssetService assetService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Asset asset, IdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, AssetExportData exportData, NewIdProvider idProvider) { asset.setCustomerId(idProvider.get(tenantId, Asset::getCustomerId)); - } - - @Override - protected Asset saveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java similarity index 68% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java index b521074bab..214e314ffd 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.exportimport.importing.impl; +import lombok.RequiredArgsConstructor; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -32,12 +34,16 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportService; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; -public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { +public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; @@ -59,18 +65,7 @@ public abstract class AbstractEntityImportService() { - @Override - public ID get(TenantId tenantId, Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return getInternalId(tenantId, idExtractor.apply(entity)); - } else { - return idExtractor.apply(existingEntity); - } - } - }); - - E savedEntity = saveEntity(tenantId, entity, existingEntity, exportData); + E savedEntity = prepareAndSave(tenantId, entity, exportData, new NewIdProvider(entity, existingEntity, importSettings)); importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); @@ -79,9 +74,7 @@ public abstract class AbstractEntityImportService idProvider) {} - - protected abstract E saveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + protected abstract E prepareAndSave(TenantId tenantId, E entity, D exportData, NewIdProvider idProvider); private void importRelations(TenantId tenantId, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) { @@ -94,7 +87,7 @@ public abstract class AbstractEntityImportService { relationService.deleteRelation(tenantId, existingRelation); }); @@ -107,14 +100,16 @@ public abstract class AbstractEntityImportService { relationService.deleteRelation(tenantId, existingRelation); }); } } - newRelations.forEach(relation -> relationService.saveRelation(tenantId, relation)); + newRelations.forEach(relation -> { + relationService.saveRelation(tenantId, relation); + }); } private ID getInternalId(TenantId tenantId, ID externalId) { @@ -128,8 +123,36 @@ public abstract class AbstractEntityImportService { - I get(TenantId tenantId, Function idExtractor); + @RequiredArgsConstructor + protected class NewIdProvider { + private final E entity; + private final E existingEntity; + private final EntityImportSettings importSettings; + + private final Set ALWAYS_UPDATE_REFERENCED_IDS = Set.of( + EntityType.RULE_CHAIN + ); + + public ID get(TenantId tenantId, Function idExtractor) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() + || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { + return getInternalId(tenantId, idExtractor.apply(entity)); + } else { + return idExtractor.apply(existingEntity); + } + } + + public Set get(TenantId tenantId, Function> listExtractor, Function idGetter, BiConsumer idSetter) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { + return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() + .peek(t -> { + idSetter.accept(t, getInternalId(tenantId, idGetter.apply(t))); + }) + .collect(Collectors.toSet()); + } else { + return listExtractor.apply(existingEntity); + } + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 22cdc10204..7b5b3e3248 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -28,12 +28,12 @@ import org.thingsboard.server.service.exportimport.exporting.data.CustomerExport @Service @TbCoreComponent @RequiredArgsConstructor -public class CustomerImportService extends AbstractEntityImportService { +public class CustomerImportService extends BaseEntityImportService { private final CustomerService customerService; @Override - protected Customer saveEntity(TenantId tenantId, Customer customer, Customer existingCustomer, CustomerExportData exportData) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, CustomerExportData exportData, NewIdProvider idProvider) { return customerService.saveCustomer(customer); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java index d78f0b40f5..2bf9a7926a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java @@ -19,31 +19,55 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + @Service @TbCoreComponent @RequiredArgsConstructor -public class DashboardImportService extends AbstractEntityImportService { +public class DashboardImportService extends BaseEntityImportService { private final DashboardService dashboardService; + // TODO [viacheslav]: improve the code @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Dashboard dashboard, IdProvider idProvider) { -// if (existingDashboard == null) { -// dashboard.setAssignedCustomers(null); // FIXME [viacheslav]: need to assign dashboard to customers ? -// } else { -// dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); -// } - } + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, DashboardExportData exportData, NewIdProvider idProvider) { + if (dashboard.getId() == null) { + Set assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); + dashboard.setAssignedCustomers(null); + dashboard = dashboardService.saveDashboard(dashboard); + for (ShortCustomerInfo customerInfo : assignedCustomers) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerInfo.getCustomerId()); + } + } else { + Set existingAssignedCustomers = dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers().stream() + .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() + .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - @Override - protected Dashboard saveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { - return dashboardService.saveDashboard(dashboard); + Set toUnassign = new HashSet<>(existingAssignedCustomers); + toUnassign.removeAll(newAssignedCustomers); + for (CustomerId customerId : toUnassign) { + dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId); + } + Set toAssign = new HashSet<>(newAssignedCustomers); + toAssign.removeAll(existingAssignedCustomers); + for (CustomerId customerId : toAssign) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + } + dashboard.setAssignedCustomers(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()); + dashboard = dashboardService.saveDashboard(dashboard); + } + return dashboard; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 8c02edec05..ee6d89b774 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -28,20 +28,16 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportDa @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceImportService extends AbstractEntityImportService { +public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Device device, IdProvider idProvider) { + protected Device prepareAndSave(TenantId tenantId, Device device, DeviceExportData exportData, NewIdProvider idProvider) { device.setCustomerId(idProvider.get(tenantId, Device::getCustomerId)); device.setDeviceProfileId(idProvider.get(tenantId, Device::getDeviceProfileId)); device.setFirmwareId(idProvider.get(tenantId, Device::getFirmwareId)); device.setSoftwareId(idProvider.get(tenantId, Device::getSoftwareId)); - } - - @Override - protected Device saveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index 9981607481..5d05bd06eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -28,20 +28,16 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileE @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceProfileImportService extends AbstractEntityImportService { +public class DeviceProfileImportService extends BaseEntityImportService { private final DeviceProfileService deviceProfileService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfileExportData exportData, NewIdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.get(tenantId, DeviceProfile::getDefaultRuleChainId)); deviceProfile.setDefaultDashboardId(idProvider.get(tenantId, DeviceProfile::getDefaultDashboardId)); deviceProfile.setFirmwareId(idProvider.get(tenantId, DeviceProfile::getFirmwareId)); deviceProfile.setSoftwareId(idProvider.get(tenantId, DeviceProfile::getSoftwareId)); - } - - @Override - protected DeviceProfile saveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 6be2fc4317..34d346ccc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.service.exportimport.importing.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; @@ -26,39 +29,47 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import java.util.Collections; +import java.util.Optional; +import java.util.UUID; + @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainImportService extends AbstractEntityImportService { +public class RuleChainImportService extends BaseEntityImportService { private final RuleChainService ruleChainService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { - } + protected RuleChain prepareAndSave(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData, NewIdProvider idProvider) { + RuleChainMetaData metaData = exportData.getMetaData(); + Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()) + .forEach(ruleNode -> { + ruleNode.setId(null); + ruleNode.setRuleChainId(null); + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); + Optional.ofNullable(ruleNodeConfig) + .flatMap(config -> Optional.ofNullable(config.get("ruleChainId")).filter(JsonNode::isTextual)) + .map(JsonNode::asText).map(UUID::fromString) + .ifPresent(otherRuleChainUuid -> { + ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( + idProvider.get(tenantId, rc -> new RuleChainId(otherRuleChainUuid)).toString() + )); + ruleNode.setConfiguration(ruleNodeConfig); + }); + }); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(tenantId, rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + }); + ruleChain.setFirstRuleNodeId(null); - @Override - protected RuleChain saveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { - ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting - if (existingRuleChain != null) { - ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); + if (ruleChain.getId() != null) { + ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); } - ruleChain = ruleChainService.saveRuleChain(ruleChain); - - RuleChainMetaData metaData = exportData.getMetaData(); - metaData.setRuleChainId(ruleChain.getId()); - metaData.getNodes().forEach(ruleNode -> { - ruleNode.setId(null); - ruleNode.setRuleChainId(null); - }); - metaData.getRuleChainConnections().forEach(ruleChainConnectionInfo -> { -// ruleChainConnectionInfo.setTargetRuleChainId(); - // TODO [viacheslav]: check if this thing is needed for "Other Rule Chain Node" - // TODO [viacheslav]: and check import of tenant rule chains - }); - ruleChainService.saveRuleChainMetaData(tenantId, metaData); - + exportData.getMetaData().setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(tenantId, exportData.getMetaData()); return ruleChain; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index f4fa052648..47ce08c4c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -238,6 +238,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.TENANT, "tenant"); entityTableMap.put(EntityType.API_USAGE_STATE, SELECT_API_USAGE_STATE); entityTableMap.put(EntityType.EDGE, "edge"); + entityTableMap.put(EntityType.RULE_CHAIN, "rule_chain"); + entityTableMap.put(EntityType.DEVICE_PROFILE, "device_profile"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 8df341fb0b..1499b7150f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -102,6 +102,7 @@ public class EntityKeyMapping { allowedEntityFieldMap.put(EntityType.WIDGET_TYPE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.API_USAGE_STATE, apiUsageStateEntityFields); + allowedEntityFieldMap.put(EntityType.DEVICE_PROFILE, Set.of(CREATED_TIME, NAME, TYPE)); entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY); entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY); From e37a075031ea450c8777048949815062b23b0687 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 1 Apr 2022 15:06:50 +0300 Subject: [PATCH 030/178] Export/import API: refactor, rename packages --- .../server/controller/AssetController.java | 4 +-- .../server/controller/BaseController.java | 12 ++++--- .../server/controller/DeviceController.java | 4 +-- .../server/controller/EdgeController.java | 4 +-- .../EntitiesExportImportController.java | 35 ++++++++++--------- .../controller/RuleChainController.java | 1 - .../service/asset/AssetBulkImportService.java | 4 +-- .../device/DeviceBulkImportService.java | 4 +-- .../service/edge/EdgeBulkImportService.java | 4 +-- .../DefaultEntitiesExportImportService.java | 15 ++++---- .../EntitiesExportImportService.java | 10 +++--- .../exporting/EntityExportService.java | 4 +-- .../exporting/EntityExportSettings.java | 2 +- .../exporting}/ExportableEntitiesService.java | 2 +- .../exporting/data/AssetExportData.java | 2 +- .../exporting/data/CustomerExportData.java | 2 +- .../exporting/data/DashboardExportData.java | 2 +- .../exporting/data/DeviceExportData.java | 2 +- .../data/DeviceProfileExportData.java | 2 +- .../exporting/data/EntityExportData.java | 6 ++-- .../exporting/data/RuleChainExportData.java | 2 +- .../exporting/impl/AssetExportService.java | 4 +-- .../impl/BaseEntityExportService.java | 16 ++++----- .../exporting/impl/CustomerExportService.java | 4 +-- .../impl/DashboardExportService.java | 4 +-- .../exporting/impl/DeviceExportService.java | 4 +-- .../impl/DeviceProfileExportService.java | 4 +-- .../impl/RuleChainExportService.java | 4 +-- .../importing/EntityImportResult.java | 2 +- .../importing/EntityImportService.java | 4 +-- .../importing/EntityImportSettings.java | 2 +- .../csv}/AbstractBulkImportService.java | 4 +-- .../importing/csv}/BulkImportColumnType.java | 2 +- .../importing/csv}/BulkImportRequest.java | 2 +- .../importing/csv}/BulkImportResult.java | 2 +- .../importing/csv}/ImportedEntityInfo.java | 2 +- .../importing/impl/AssetImportService.java | 4 +-- .../impl/BaseEntityImportService.java | 16 ++++----- .../importing/impl/CustomerImportService.java | 4 +-- .../impl/DashboardImportService.java | 4 +-- .../importing/impl/DeviceImportService.java | 4 +-- .../impl/DeviceProfileImportService.java | 4 +-- .../impl/RuleChainImportService.java | 6 ++-- .../server/dao/rule/RuleChainService.java | 1 + .../server/common/data/id/IdBased.java | 35 ++++++++++--------- .../server/common/data/rule/RuleChain.java | 4 +-- .../server/dao/ExportableEntityDao.java | 1 - 47 files changed, 136 insertions(+), 130 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/DefaultEntitiesExportImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/EntitiesExportImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/EntityExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/EntityExportSettings.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync/exporting}/ExportableEntitiesService.java (95%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/AssetExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/CustomerExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DashboardExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DeviceExportData.java (94%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DeviceProfileExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/EntityExportData.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/RuleChainExportData.java (94%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/AssetExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/BaseEntityExportService.java (79%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/CustomerExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DashboardExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DeviceProfileExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportSettings.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/AbstractBulkImportService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportColumnType.java (97%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportRequest.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportResult.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/ImportedEntityInfo.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/AssetImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/BaseEntityImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/CustomerImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DeviceImportService.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DeviceProfileImportService.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/RuleChainImportService.java (94%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 94c714e24f..b05e32efe7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -54,8 +54,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 7416444104..382929e1e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); @@ -951,7 +951,7 @@ public abstract class BaseController { otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); notifyEdgeService = true; break; - case RULE_CHAIN: // FIXME: events for rule chain metadata + case RULE_CHAIN: RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); if (RuleChainType.CORE.equals(ruleChainType)) { tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), @@ -974,8 +974,12 @@ public abstract class BaseController { throw new UnsupportedOperationException(); } - entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, - isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + try { + logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, + isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } if (notifyEdgeService) { sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index d2ffafa73e..feb58edb7a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,8 +74,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index f3d73f13e5..8f45427603 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -55,8 +55,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 411a5ec00e..f158ff15e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -42,12 +42,12 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -103,12 +103,12 @@ public class EntitiesExportImportController extends BaseController { } } - @PostMapping(value = "/export/{entityType}", params = "ids") - public List>> exportAllEntitiesByEntityType(@PathVariable EntityType entityType, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + @PostMapping(value = "/export/{entityType}") + public List>> exportEntitiesByEntityType(@PathVariable EntityType entityType, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { TenantId tenantId = getTenantId(); CustomerId customerId = toCustomerId(customerUuid); @@ -236,11 +236,12 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { for (EntityExportData> exportData : exportDataList) { - ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getEntity().getId()); if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.WRITE, existingEntity.getId(), existingEntity); } else { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + exportData.getEntity().setTenantId(user.getTenantId()); + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE, null, exportData.getEntity()); } List relations = new LinkedList<>(); @@ -252,9 +253,9 @@ public class EntitiesExportImportController extends BaseController { } for (EntityRelation relation : relations) { EntityId otherEntityId = null; - if (!relation.getFrom().equals(exportData.getMainEntity().getId())) { + if (!relation.getFrom().equals(exportData.getEntity().getId())) { otherEntityId = relation.getFrom(); - } else if (!relation.getTo().equals(exportData.getMainEntity().getId())) { + } else if (!relation.getTo().equals(exportData.getEntity().getId())) { otherEntityId = relation.getTo(); } if (otherEntityId != null) { diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index aba074ac60..cb59951ac5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -279,7 +279,6 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); - tbClusterService.broadcastEntityStateChangeEvent(savedRuleChain.getTenantId(), savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, null, ActionType.ADDED, null); diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index 4c75f7d45d..cfefcafbf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index cd655a2467..3aac9c3641 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index 615a7cddd2..b09ac532df 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 8d85355849..f399dbdbea 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -24,12 +24,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportService; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.Collection; import java.util.Comparator; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 682db6ba21..0fbe75a866 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 929092e90e..6b57cbb557 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting; +package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java index f4eea9913a..36552b2c12 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting; +package org.thingsboard.server.service.sync.exporting; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index bb8f30c4e1..2135d2aca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java index 4c4872760b..4031ec7062 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java index 1fc820814e..08888f4a35 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java index 6ef45fc531..6f101816c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 5f5d8e8f72..977bd22f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java index def6b7edbe..7b71749be4 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index b9cc8c71a3..51b5615c9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType") @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), @@ -43,7 +43,7 @@ import java.util.List; @Data public abstract class EntityExportData> { - private E mainEntity; + private E entity; private List inboundRelations; private List outboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index f92616fead..9f59aa4401 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java index c22b345299..b4f506a1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 68c78e87c3..a3a83196bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -23,10 +23,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.List; @@ -41,9 +41,9 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java index 1f23e8e7d2..c90d21e573 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java index 17d593157b..32584ca917 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index b8c7e65b34..d31a859013 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java index 24b5f8891e..b3fd65363b 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java index f1b9a27f23..846d4006d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java index cc44670757..a79bfa2478 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import lombok.Data; import org.thingsboard.server.common.data.ExportableEntity; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index 8697b04f59..19947e8917 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java index c3cf0b531d..9e4afcc560 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java index b7b740f190..06cd7a93be 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.importing.BulkImportRequest.ColumnMapping; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java index 96075eb4c8..beafe5e95f 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java index 9f8195ca45..f600289ab6 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java index 651aedeb0b..da12a6baeb 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java index 45c2551be2..68e7a49b18 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 04e5a5eeca..b8328a43c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 214e314ffd..608ffe2acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.apache.commons.collections.CollectionUtils; @@ -28,11 +28,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportService; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.Collections; import java.util.LinkedList; @@ -53,7 +53,7 @@ public abstract class BaseEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { - E entity = exportData.getMainEntity(); + E entity = exportData.getEntity(); E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); entity.setExternalId(entity.getId()); @@ -118,7 +118,7 @@ public abstract class BaseEntityImportService entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); if (entity == null) { - throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); + throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } return entity.getId(); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 7b5b3e3248..5ad7161486 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 2bf9a7926a..9e7cffd272 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; import java.util.HashSet; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index ee6d89b774..eafe9c0fdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index 5d05bd06eb..61b1ea16dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 34d346ccc9..5dfc31e97a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import java.util.Collections; import java.util.Optional; @@ -70,7 +70,7 @@ public class RuleChainImportService extends BaseEntityImportService implements HasId { protected I id; - - public IdBased() { - super(); - } - - public IdBased(I id) { - super(); - this.id = id; - } + + public IdBased() { + super(); + } + + public IdBased(I id) { + super(); + this.id = id; + } @JsonSetter - public void setId(I id) { - this.id = id; - } + public void setId(I id) { + this.id = id; + } - public I getId() { - return id; - } + public I getId() { + return id; + } - @JsonIgnore - public UUID getUuidId() { + @JsonIgnore + public UUID getUuidId() { if (id != null) { return id.getId(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 74186b49bf..eca9e331cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -57,11 +57,11 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @ApiModelProperty(position = 9, value = "Reserved for future usage. The actual list of rule nodes and their relations is stored in the database separately.") private transient JsonNode configuration; + private RuleChainId externalId; + @JsonIgnore private byte[] configurationBytes; - private RuleChainId externalId; - public RuleChain() { super(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 2fb301c092..5b2b5a2366 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -25,7 +25,6 @@ public interface ExportableEntityDao { T findByTenantIdAndId(UUID tenantId, UUID id); - EntityType getEntityType(); } From 4dcd037a61eeadf576a59d3e8275596b3699a612 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 1 Apr 2022 15:24:22 +0300 Subject: [PATCH 031/178] Initial tests for export/import --- .../server/controller/AbstractWebTest.java | 1 - ...aseEntitiesExportImportControllerTest.java | 166 +++++++++++++ ...EntitiesExportImportControllerSqlTest.java | 228 ++++++++++++++++++ 3 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index cc561e31a8..04a2c5f351 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -605,7 +605,6 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected T readResponse(ResultActions result, TypeReference type) throws Exception { byte[] content = result.andReturn().getResponse().getContentAsByteArray(); - ObjectMapper mapper = new ObjectMapper(); return mapper.readerFor(type).readValue(content); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java new file mode 100644 index 0000000000..cf88cf3d0d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.fasterxml.jackson.databind.node.TextNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.rule.RuleChainService; + +import java.util.Arrays; +import java.util.Collections; + +public abstract class BaseEntitiesExportImportControllerTest extends AbstractControllerTest { + + @Autowired + protected DeviceService deviceService; + @Autowired + protected DeviceProfileService deviceProfileService; + @Autowired + protected AssetService assetService; + @Autowired + protected CustomerService customerService; + @Autowired + protected RuleChainService ruleChainService; + @Autowired + protected DashboardService dashboardService; + + protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName(name); + device.setLabel("lbl"); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + device.setDeviceData(deviceData); + return deviceService.saveDevice(device); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setName(name); + deviceProfile.setDescription("dscrptn"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCustomerId(customerId); + asset.setType(type); + asset.setName(name); + asset.setLabel("lbl"); + asset.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return assetService.saveAsset(asset); + } + + protected Customer createCustomer(TenantId tenantId, String name) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(name); + customer.setCountry("ua"); + customer.setAddress("abb"); + customer.setEmail("ccc@aa.org"); + customer.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return customerService.saveCustomer(customer); + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setTitle(name); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + dashboard.setImage("abvregewrg"); + dashboard.setMobileHide(true); + dashboard = dashboardService.saveDashboard(dashboard); + if (customerId != null) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + return dashboardService.findDashboardById(tenantId, dashboard.getId()); + } + return dashboard; + } + + protected RuleChain createRuleChain(TenantId tenantId, String name) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java new file mode 100644 index 0000000000..d986f5f9ad --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller.sql; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import lombok.SneakyThrows; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.ResultActions; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.controller.BaseEntitiesExportImportControllerTest; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; + +import java.util.List; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { + + @Autowired + private TenantService tenantService; + @Autowired + private DeviceCredentialsService deviceCredentialsService; + @Autowired + private ExportableEntitiesService exportableEntitiesService; + + private TenantId tenantId1; + private User tenantAdmin1; + + private TenantId tenantId2; + private User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } + + @Test + public void testExportImport_singleEntityOneByOne_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); + testExportImportBetweenTenants(asset, importedAsset -> { + assertThat(importedAsset.getName()).isEqualTo(asset.getName()); + assertThat(importedAsset.getType()).isEqualTo(asset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(asset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(asset.getAdditionalInfo()); + }); + + Customer customer = createCustomer(tenantId1, "Customer of tenant 1"); + testExportImportBetweenTenants(customer, importedCustomer -> { + assertThat(importedCustomer.getTitle()).isEqualTo(customer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(customer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(customer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(customer.getEmail()); + }); + + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); + DeviceProfileId importedDeviceProfileId = testExportImportBetweenTenants(deviceProfile, importedDeviceProfile -> { + assertThat(importedDeviceProfile.getName()).isEqualTo(deviceProfile.getName()); + assertThat(importedDeviceProfile.getType()).isEqualTo(deviceProfile.getType()); + assertThat(importedDeviceProfile.getTransportType()).isEqualTo(deviceProfile.getTransportType()); + assertThat(importedDeviceProfile.getProfileData()).isEqualTo(deviceProfile.getProfileData()); + assertThat(importedDeviceProfile.getDescription()).isEqualTo(deviceProfile.getDescription()); + }).getSavedEntity().getId(); + + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + this.testExportImportBetweenTenants(device, deviceExportData -> { + assertThat(deviceExportData.getCredentials()).isEqualTo(credentials); + }, importedDevice -> { + assertThat(importedDevice.getName()).isEqualTo(device.getName()); + assertThat(importedDevice.getType()).isEqualTo(device.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(device.getDeviceData()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfileId); + assertThat(importedDevice.getLabel()).isEqualTo(device.getLabel()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(credentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + }); + + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + this.testExportImportBetweenTenants(ruleChain, ruleChainExportData -> { + assertThat(ruleChainExportData.getMetaData()).isEqualTo(metaData); + }, importedRuleChain -> { + assertThat(importedRuleChain.getType()).isEqualTo(ruleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(ruleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(ruleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(ruleChain.getConfiguration()); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + assertThat(importedMetaData.getConnections()).isEqualTo(metaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(metaData.getFirstNodeIndex()); + for (int i = 0; i < metaData.getNodes().size(); i++) { + RuleNode initialNode = metaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + }); + + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard of tenant 1"); + testExportImportBetweenTenants(dashboard, importedDashboard -> { + assertThat(importedDashboard.getTitle()).isEqualTo(dashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(dashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(dashboard.isMobileHide()); + }); + } + + private , D extends EntityExportData> EntityImportResult testExportImportBetweenTenants(E entity, Consumer exportDataRequirements, Consumer importedEntityRequirements) throws Exception { + logInAsTenantAdmin1(); + D exportData = readResponse(doPost("/api/entities/export/" + entity.getId().getEntityType() + "/" + entity.getId()) + .andExpect(status().isOk()), new TypeReference() {}); + assertThat(exportData.getEntity()).isEqualTo(entity); + exportDataRequirements.accept(exportData); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + E importedEntity = (E) exportableEntitiesService.findEntityById(tenantId2, importResult.getSavedEntity().getId()); + assertThat(importedEntity).isNotNull(); + assertThat(importResult.getSavedEntity()).isEqualTo(importedEntity); + assertThat(importResult.getOldEntity()).isNull(); + + assertThat(importedEntity.getId()).isNotEqualTo(entity.getId()); + assertThat(importedEntity.getExternalId()).isEqualTo(entity.getId()); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + importedEntityRequirements.accept(importedEntity); + return importResult; + } + + private > EntityImportResult testExportImportBetweenTenants(E entity, Consumer importedEntityRequirements) throws Exception { + return testExportImportBetweenTenants(entity, exportData -> {}, importedEntityRequirements); + } + + + @SneakyThrows + private > List> importEntities(List> exportDataList) { + String requestJson = mapper.writerFor(new TypeReference>>() {}).writeValueAsString(exportDataList); + ResultActions resultActions = doPost("/api/entities/import", (Object) requestJson); + + try { + String responseJson = resultActions.andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + JavaType type = mapper.getTypeFactory().constructCollectionType(List.class, + mapper.getTypeFactory().constructParametricType(EntityImportResult.class, exportDataList.get(0).getEntity().getClass())); + return mapper.readValue(responseJson, type); + } catch (AssertionError e) { + throw new AssertionError(readResponse(resultActions, String.class), e); + } + } + + private void logInAsTenantAdmin1() throws Exception { + login(tenantAdmin1.getEmail(), "12345678"); + } + + private void logInAsTenantAdmin2() throws Exception { + login(tenantAdmin2.getEmail(), "12345678"); + } + +} From 2a77c90d13871ac091e9840b1ba91557727e47ca Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 15:09:46 +0300 Subject: [PATCH 032/178] More tests for export/import api; refactoring --- .../DefaultEntitiesExportImportService.java | 29 +- .../exporting/ExportableEntitiesService.java | 6 +- .../sync/exporting/TenantEntitiesService.java | 27 ++ .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 8 +- ...EntitiesExportImportControllerSqlTest.java | 350 +++++++++++++----- .../server/dao/ExportableEntityDao.java | 8 +- .../dao/ExportableEntityRepository.java | 4 +- .../server/dao/TenantEntityDao.java | 11 +- .../server/dao/TenantEntityRepository.java | 26 ++ .../server/dao/asset/AssetDao.java | 2 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 +- .../server/dao/rule/RuleChainDao.java | 2 +- .../dao/sql/customer/CustomerRepository.java | 1 - .../dao/sql/device/JpaDeviceProfileDao.java | 5 + .../server/dao/sql/ota/JpaOtaPackageDao.java | 18 + .../dao/sql/ota/OtaPackageRepository.java | 3 +- .../dao/sql/rule/RuleChainRepository.java | 2 - .../server/dao/sql/user/JpaUserDao.java | 13 + .../server/dao/sql/user/UserRepository.java | 5 +- .../thingsboard/server/dao/user/UserDao.java | 2 +- 24 files changed, 411 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index f399dbdbea..eeaef01f63 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -20,9 +20,12 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; @@ -37,7 +40,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; @Service @@ -46,7 +48,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); + private final Map> daos = new HashMap<>(); protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -85,16 +87,22 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { - ExportableEntityDao dao = getDao(id.getEntityType()); + public & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id) { + TenantEntityDao dao = (TenantEntityDao) getDao(id.getEntityType()); return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); } @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { - ExportableEntityDao dao = getDao(externalId.getEntityType()); - return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) - .orElseGet(() -> findEntityById(tenantId, externalId)); + EntityType entityType = externalId.getEntityType(); + if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { + ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); + E entity = dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + if (entity != null) { + return entity; + } + } + return findEntityById(tenantId, externalId); } @@ -114,15 +122,14 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } - @SuppressWarnings("unchecked") - private ExportableEntityDao getDao(EntityType entityType) { - return (ExportableEntityDao) daos.get(entityType); + private TenantEntityDao getDao(EntityType entityType) { + return daos.get(entityType); } @Autowired private void setServices(Collection> exportServices, Collection> importServices, - Collection> daos) { + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 2135d2aca1..1c1cf6bf7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,12 +16,12 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntitiesService { - - , I extends EntityId> E findEntityById(TenantId tenantId, I id); +public interface ExportableEntitiesService extends TenantEntitiesService { , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java new file mode 100644 index 0000000000..e3e25c21e8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting; + +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantEntitiesService { + + & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 9e7cffd272..5525ba8e6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -27,7 +27,9 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import java.util.Collections; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -49,8 +51,8 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers().stream() - .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) + .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index eafe9c0fdb..2dcf4f7e7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -38,7 +38,13 @@ public class DeviceImportService extends BaseEntityImportService { - assertThat(importedAsset.getName()).isEqualTo(asset.getName()); - assertThat(importedAsset.getType()).isEqualTo(asset.getType()); - assertThat(importedAsset.getLabel()).isEqualTo(asset.getLabel()); - assertThat(importedAsset.getAdditionalInfo()).isEqualTo(asset.getAdditionalInfo()); - }); - - Customer customer = createCustomer(tenantId1, "Customer of tenant 1"); - testExportImportBetweenTenants(customer, importedCustomer -> { - assertThat(importedCustomer.getTitle()).isEqualTo(customer.getTitle()); - assertThat(importedCustomer.getCountry()).isEqualTo(customer.getCountry()); - assertThat(importedCustomer.getAddress()).isEqualTo(customer.getAddress()); - assertThat(importedCustomer.getEmail()).isEqualTo(customer.getEmail()); - }); + EntityExportData exportData = exportSingleEntity(asset.getId()); + assertThat(exportData.getEntity()).isEqualTo(asset); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); - DeviceProfileId importedDeviceProfileId = testExportImportBetweenTenants(deviceProfile, importedDeviceProfile -> { - assertThat(importedDeviceProfile.getName()).isEqualTo(deviceProfile.getName()); - assertThat(importedDeviceProfile.getType()).isEqualTo(deviceProfile.getType()); - assertThat(importedDeviceProfile.getTransportType()).isEqualTo(deviceProfile.getTransportType()); - assertThat(importedDeviceProfile.getProfileData()).isEqualTo(deviceProfile.getProfileData()); - assertThat(importedDeviceProfile.getDescription()).isEqualTo(deviceProfile.getDescription()); - }).getSavedEntity().getId(); + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, asset, tenantId2, importResult); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportSingleAsset_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); + EntityExportData exportData = exportSingleEntity(asset.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, asset, tenantId1, importResult); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_withCustomer_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + EntityExportData customerExportData = exportSingleEntity(customer.getId()); + EntityExportData assetExportData = exportSingleEntity(asset.getId()); + + logInAsTenantAdmin2(); + Customer importedCustomer = importEntities(List.of(customerExportData)).get(0).getSavedEntity(); + Asset importedAsset = importEntities(List.of(assetExportData)).get(0).getSavedEntity(); + + assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); + } + + @Test + public void testExportImportAsset_withCustomer_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + Asset importedAsset = this.importEntities(List.of(exportSingleEntity(asset.getId()))).get(0).getSavedEntity(); + + assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); + } + + private void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + + @Test + public void testExportImportSingleCustomer_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); + EntityExportData exportData = exportSingleEntity(customer.getId()); + assertThat(exportData.getEntity()).isEqualTo(customer); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, customer, tenantId2, importResult); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + @Test + public void testExportImportSingleCustomer_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); + EntityExportData exportData = exportSingleEntity(customer.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, customer, tenantId1, importResult); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + private void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + + + @Test + public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); - this.testExportImportBetweenTenants(device, deviceExportData -> { - assertThat(deviceExportData.getCredentials()).isEqualTo(credentials); - }, importedDevice -> { - assertThat(importedDevice.getName()).isEqualTo(device.getName()); - assertThat(importedDevice.getType()).isEqualTo(device.getType()); - assertThat(importedDevice.getDeviceData()).isEqualTo(device.getDeviceData()); - assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfileId); - assertThat(importedDevice.getLabel()).isEqualTo(device.getLabel()); - - DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); - assertThat(importedCredentials.getCredentialsId()).isEqualTo(credentials.getCredentialsId()); - assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); - assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); - }); + EntityExportData profileExportData = exportSingleEntity(deviceProfile.getId()); + assertThat(profileExportData.getEntity()).isEqualTo(deviceProfile); + + EntityExportData deviceExportData = exportSingleEntity(device.getId()); + assertThat(deviceExportData.getEntity()).isEqualTo(device); + assertThat(((DeviceExportData) deviceExportData).getCredentials()).isEqualTo(credentials); + DeviceCredentials exportedCredentials = ((DeviceExportData) deviceExportData).getCredentials(); + exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); + + logInAsTenantAdmin2(); + EntityImportResult profileImportResult = importEntities(List.of(profileExportData)).get(0); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); + checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); + + + EntityImportResult deviceImportResult = importEntities(List.of(deviceExportData)).get(0); + Device importedDevice = deviceImportResult.getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); + checkImportedDeviceData(device, importedDevice); + + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getId()).isNotEqualTo(credentials.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(exportedCredentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + } + + @Test + public void testExportImportDevice_sameTenant() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile v1.0"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData deviceExportData = exportSingleEntity(device.getId()); + + EntityImportResult importResult = importEntities(List.of(deviceExportData)).get(0); + Device importedDevice = importResult.getSavedEntity(); + + checkImportedEntity(tenantId1, device, tenantId1, importResult); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); + assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + } + + private void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + + private void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + + @Test + public void testExportImportSingleRuleChain_betweenTenants() throws Exception { + logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); - this.testExportImportBetweenTenants(ruleChain, ruleChainExportData -> { - assertThat(ruleChainExportData.getMetaData()).isEqualTo(metaData); - }, importedRuleChain -> { - assertThat(importedRuleChain.getType()).isEqualTo(ruleChain.getType()); - assertThat(importedRuleChain.getName()).isEqualTo(ruleChain.getName()); - assertThat(importedRuleChain.isDebugMode()).isEqualTo(ruleChain.isDebugMode()); - assertThat(importedRuleChain.getConfiguration()).isEqualTo(ruleChain.getConfiguration()); - RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); - assertThat(importedMetaData.getConnections()).isEqualTo(metaData.getConnections()); - assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(metaData.getFirstNodeIndex()); - for (int i = 0; i < metaData.getNodes().size(); i++) { - RuleNode initialNode = metaData.getNodes().get(i); - RuleNode importedNode = importedMetaData.getNodes().get(i); - assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); - assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); - assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); - assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); - } - }); - - Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard of tenant 1"); - testExportImportBetweenTenants(dashboard, importedDashboard -> { - assertThat(importedDashboard.getTitle()).isEqualTo(dashboard.getTitle()); - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - assertThat(importedDashboard.getImage()).isEqualTo(dashboard.getImage()); - assertThat(importedDashboard.isMobileHide()).isEqualTo(dashboard.isMobileHide()); - }); - } - - private , D extends EntityExportData> EntityImportResult testExportImportBetweenTenants(E entity, Consumer exportDataRequirements, Consumer importedEntityRequirements) throws Exception { + + EntityExportData exportData = exportSingleEntity(ruleChain.getId()); + assertThat(exportData.getEntity()).isEqualTo(ruleChain); + assertThat(((RuleChainExportData) exportData).getMetaData()).isEqualTo(metaData); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + @Test + public void testExportImportSingleRuleChain_sameTenant() throws Exception { logInAsTenantAdmin1(); - D exportData = readResponse(doPost("/api/entities/export/" + entity.getId().getEntityType() + "/" + entity.getId()) - .andExpect(status().isOk()), new TypeReference() {}); - assertThat(exportData.getEntity()).isEqualTo(entity); - exportDataRequirements.accept(exportData); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + + EntityExportData exportData = exportSingleEntity(ruleChain.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + private void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + + @Test + public void testExportImportSingleDashboard_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + assertThat(exportData.getEntity()).isEqualTo(dashboard); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } - E importedEntity = (E) exportableEntitiesService.findEntityById(tenantId2, importResult.getSavedEntity().getId()); - assertThat(importedEntity).isNotNull(); - assertThat(importResult.getSavedEntity()).isEqualTo(importedEntity); - assertThat(importResult.getOldEntity()).isNull(); + @Test + public void testExportImportSingleDashboard_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); - assertThat(importedEntity.getId()).isNotEqualTo(entity.getId()); - assertThat(importedEntity.getExternalId()).isEqualTo(entity.getId()); - assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); - importedEntityRequirements.accept(importedEntity); - return importResult; + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private > EntityImportResult testExportImportBetweenTenants(E entity, Consumer importedEntityRequirements) throws Exception { - return testExportImportBetweenTenants(entity, exportData -> {}, importedEntityRequirements); + private void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + + + private > void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { + E importedEntity = importResult.getSavedEntity(); + + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + assertThat(importResult.getOldEntity()).isEqualTo(initialEntity); + } } @@ -217,6 +391,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } } + private , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { + return readResponse(doPost("/api/entities/export/" + entityId.getEntityType() + "/" + entityId.getId().toString()) + .andExpect(status().isOk()), new TypeReference>() {}); + } + + private void logInAsTenantAdmin1() throws Exception { login(tenantAdmin1.getEmail(), "12345678"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 5b2b5a2366..9f2cd51952 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -15,16 +15,12 @@ */ package org.thingsboard.server.dao; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; import java.util.UUID; -public interface ExportableEntityDao { +public interface ExportableEntityDao> extends TenantEntityDao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - T findByTenantIdAndId(UUID tenantId, UUID id); - - EntityType getEntityType(); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java index b6a303a941..1ce43ffe12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -17,10 +17,8 @@ package org.thingsboard.server.dao; import java.util.UUID; -public interface ExportableEntityRepository { +public interface ExportableEntityRepository extends TenantEntityRepository { D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - D findByTenantIdAndId(UUID tenantId, UUID id); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 17367c2cc7..53ffb7882a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,9 +15,18 @@ */ package org.thingsboard.server.dao; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.TenantId; -public interface TenantEntityDao { +import java.util.UUID; + +public interface TenantEntityDao { Long countByTenantId(TenantId tenantId); + + T findByTenantIdAndId(UUID tenantId, UUID id); + + EntityType getEntityType(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java new file mode 100644 index 0000000000..8997f65ca8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface TenantEntityRepository { + + D findByTenantIdAndId(UUID tenantId, UUID id); + + Long countByTenantId(UUID tenantId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 495886bcc4..c7cf6f38e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index 89672d2a73..df589711b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -29,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 90353f49b3..ef6a739391 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 90f2b218a7..688e7246d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index 445210ca3b..a24df5794c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,6 +21,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; -public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { +public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, TenantEntityDao { Long sumDataSizeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 59f287eadd..47f69cc99d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 1300fcdfa5..893f22503d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -38,5 +38,4 @@ public interface CustomerRepository extends JpaRepository, CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); - Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 539377a647..26e49e9c66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,6 +116,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao { +public interface OtaPackageRepository extends JpaRepository, TenantEntityRepository { @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 888951778c..2b1bca32b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -63,8 +63,6 @@ public interface RuleChainRepository extends JpaRepository findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 6b2de9872a..42d331cab5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -96,4 +98,15 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public User findByTenantIdAndId(UUID tenantId, UUID id) { + return DaoUtil.getData(userRepository.findByTenantIdAndId(tenantId, id)); + } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 51f6138cc1..98b7cf0d00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -21,6 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.TenantEntityRepository; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.UUID; @@ -28,7 +30,7 @@ import java.util.UUID; /** * @author Valerii Sosliuk */ -public interface UserRepository extends JpaRepository { +public interface UserRepository extends JpaRepository, TenantEntityRepository { UserEntity findByEmail(String email); @@ -47,5 +49,4 @@ public interface UserRepository extends JpaRepository { @Param("searchText") String searchText, Pageable pageable); - Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index b906b3278d..7e27c1bf00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object From 9b8fca99733b70e732f17085cd646c085a120703 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 19:22:16 +0300 Subject: [PATCH 033/178] Export/import api improvements --- .../server/controller/BaseController.java | 6 ++--- .../DefaultEntitiesExportImportService.java | 23 +++++++++------- .../exporting/ExportableEntitiesService.java | 5 ++-- .../sync/exporting/TenantEntitiesService.java | 27 ------------------- .../importing/impl/AssetImportService.java | 1 + .../impl/BaseEntityImportService.java | 1 - .../importing/impl/CustomerImportService.java | 1 + .../impl/DashboardImportService.java | 1 + .../importing/impl/DeviceImportService.java | 1 + .../impl/DeviceProfileImportService.java | 1 + .../impl/RuleChainImportService.java | 1 + ...EntitiesExportImportControllerSqlTest.java | 6 ++--- .../server/common/data/ExportableEntity.java | 6 +---- .../java/org/thingsboard/server/dao/Dao.java | 3 +++ .../server/dao/ExportableEntityDao.java | 2 +- .../dao/ExportableEntityRepository.java | 2 +- .../server/dao/TenantEntityDao.java | 11 +------- .../server/dao/TenantEntityRepository.java | 26 ------------------ .../server/dao/asset/AssetDao.java | 2 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 +- .../server/dao/rule/RuleChainDao.java | 4 +-- .../server/dao/sql/asset/JpaAssetDao.java | 5 ---- .../dao/sql/customer/CustomerRepository.java | 1 + .../dao/sql/customer/JpaCustomerDao.java | 5 ---- .../dao/sql/dashboard/JpaDashboardDao.java | 5 ---- .../server/dao/sql/device/JpaDeviceDao.java | 5 ---- .../dao/sql/device/JpaDeviceProfileDao.java | 10 ------- .../server/dao/sql/ota/JpaOtaPackageDao.java | 11 -------- .../dao/sql/ota/OtaPackageRepository.java | 3 +-- .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ---- .../dao/sql/rule/RuleChainRepository.java | 2 ++ .../server/dao/sql/user/JpaUserDao.java | 13 --------- .../server/dao/sql/user/UserRepository.java | 5 ++-- .../thingsboard/server/dao/user/UserDao.java | 2 +- 37 files changed, 47 insertions(+), 163 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 382929e1e7..1f683c7874 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected , I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); @@ -954,7 +954,7 @@ public abstract class BaseController { case RULE_CHAIN: RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); if (RuleChainType.CORE.equals(ruleChainType)) { - tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), + tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedEntity.getId(), isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } if (RuleChainType.EDGE.equals(ruleChainType)) { @@ -981,7 +981,7 @@ public abstract class BaseController { log.error("Failed to log entity action", e); } if (notifyEdgeService) { - sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + sendEntityNotificationMsg(user.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index eeaef01f63..ef9da1f89d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -48,7 +49,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); + private final Map> daos = new HashMap<>(); protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -86,12 +87,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Override - public & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id) { - TenantEntityDao dao = (TenantEntityDao) getDao(id.getEntityType()); - return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); - } - @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { EntityType entityType = externalId.getEntityType(); @@ -105,6 +100,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return findEntityById(tenantId, externalId); } + @Override + public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + Dao dao = (Dao) getDao(id.getEntityType()); + return dao.findById(tenantId, id.getId()); + } + @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { @@ -122,14 +123,14 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } - private TenantEntityDao getDao(EntityType entityType) { + private Dao getDao(EntityType entityType) { return daos.get(entityType); } @Autowired private void setServices(Collection> exportServices, Collection> importServices, - Collection> daos) { + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); @@ -137,7 +138,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS this.importServices.put(entityImportService.getEntityType(), entityImportService); }); daos.forEach(dao -> { - this.daos.put(dao.getEntityType(), dao); + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } }); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 1c1cf6bf7f..9a47e98920 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,13 +16,14 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntitiesService extends TenantEntitiesService { +public interface ExportableEntitiesService { , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityById(TenantId tenantId, I id); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java deleted file mode 100644 index e3e25c21e8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting; - -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; - -public interface TenantEntitiesService { - - & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index b8328a43c3..b9013fd549 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -34,6 +34,7 @@ public class AssetImportService extends BaseEntityImportService assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 2dcf4f7e7f..f35a20e312 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -34,6 +34,7 @@ public class DeviceImportService extends BaseEntityImportService { diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 464b5da401..f114af6625 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -28,11 +28,10 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; @@ -52,7 +51,6 @@ import org.thingsboard.server.service.sync.importing.EntityImportResult; import java.util.List; -import static org.assertj.core.api.Assertions.as; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -358,7 +356,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } - private > void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { + private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { E importedEntity = importResult.getSavedEntity(); assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index f1e8c3754e..50569b7a02 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -17,9 +17,8 @@ package org.thingsboard.server.common.data; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntity extends HasId, HasTenantId, HasName { +public interface ExportableEntity extends HasId, HasName { I getId(); void setId(I id); @@ -27,7 +26,4 @@ public interface ExportableEntity extends HasId, HasTenan I getExternalId(); void setExternalId(I externalId); - TenantId getTenantId(); - void setTenantId(TenantId tenantId); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 932a6e5ec1..038be61d21 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -40,4 +41,6 @@ public interface Dao { void removeAllByIds(Collection ids); + default EntityType getEntityType() { return null; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 9f2cd51952..7e4d00286c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.ExportableEntity; import java.util.UUID; -public interface ExportableEntityDao> extends TenantEntityDao { +public interface ExportableEntityDao> extends Dao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java index 1ce43ffe12..6e2d2e115a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -17,7 +17,7 @@ package org.thingsboard.server.dao; import java.util.UUID; -public interface ExportableEntityRepository extends TenantEntityRepository { +public interface ExportableEntityRepository { D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 53ffb7882a..17367c2cc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,18 +15,9 @@ */ package org.thingsboard.server.dao; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.TenantId; -import java.util.UUID; - -public interface TenantEntityDao { +public interface TenantEntityDao { Long countByTenantId(TenantId tenantId); - - T findByTenantIdAndId(UUID tenantId, UUID id); - - EntityType getEntityType(); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java deleted file mode 100644 index 8997f65ca8..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import java.util.UUID; - -public interface TenantEntityRepository { - - D findByTenantIdAndId(UUID tenantId, UUID id); - - Long countByTenantId(UUID tenantId); - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index c7cf6f38e9..495886bcc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index df589711b9..89672d2a73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -29,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index ef6a739391..90353f49b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 688e7246d4..90f2b218a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index a24df5794c..445210ca3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,6 +21,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; -public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, TenantEntityDao { +public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { Long sumDataSizeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 47f69cc99d..1d6c1bf29a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. @@ -80,6 +80,4 @@ public interface RuleChainDao extends Dao, ExportableEntityDao findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name); - RuleChain findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index cd4fefac48..14788f8b38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -214,11 +214,6 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); } - @Override - public Asset findByTenantIdAndId(UUID tenantId, UUID id) { - return DaoUtil.getData(assetRepository.findByTenantIdAndId(tenantId, id)); - } - @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 893f22503d..1300fcdfa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -38,4 +38,5 @@ public interface CustomerRepository extends JpaRepository, CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); + Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 9cf74f1f33..17e1625a29 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -75,11 +75,6 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); } - @Override - public Device findByTenantIdAndId(UUID tenantId, UUID id) { - return findDeviceByTenantIdAndId(TenantId.fromUUID(tenantId), id); - } - @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 26e49e9c66..23e564b735 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,16 +116,6 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao, TenantEntityRepository { +public interface OtaPackageRepository extends JpaRepository { @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index e2d8aafef3..0d5fea6f98 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -114,11 +114,6 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 42d331cab5..6b2de9872a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -18,8 +18,6 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -98,15 +96,4 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } - - @Override - public User findByTenantIdAndId(UUID tenantId, UUID id) { - return DaoUtil.getData(userRepository.findByTenantIdAndId(tenantId, id)); - } - - @Override - public EntityType getEntityType() { - return EntityType.USER; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 98b7cf0d00..51f6138cc1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -21,8 +21,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.TenantEntityRepository; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.UUID; @@ -30,7 +28,7 @@ import java.util.UUID; /** * @author Valerii Sosliuk */ -public interface UserRepository extends JpaRepository, TenantEntityRepository { +public interface UserRepository extends JpaRepository { UserEntity findByEmail(String email); @@ -49,4 +47,5 @@ public interface UserRepository extends JpaRepository, TenantE @Param("searchText") String searchText, Pageable pageable); + Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index 7e27c1bf00..b906b3278d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object From 18b2fd4664b2cb5a54b671421d577fd6e638d5a3 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 5 Apr 2022 15:01:26 +0300 Subject: [PATCH 034/178] Fix permission checks for export/import api --- .../EntitiesExportImportController.java | 94 +++++-------------- .../DefaultEntitiesExportImportService.java | 69 +++++++++----- .../sync/EntitiesExportImportService.java | 9 +- .../sync/exporting/EntityExportService.java | 5 +- .../exporting/ExportableEntitiesService.java | 14 ++- .../impl/BaseEntityExportService.java | 27 ++++-- .../sync/importing/EntityImportService.java | 5 +- .../importing/impl/AssetImportService.java | 8 +- .../impl/BaseEntityImportService.java | 92 ++++++++++++------ .../importing/impl/CustomerImportService.java | 6 +- .../impl/DashboardImportService.java | 10 +- .../importing/impl/DeviceImportService.java | 14 ++- .../impl/DeviceProfileImportService.java | 14 ++- .../impl/RuleChainImportService.java | 11 ++- .../server/dao/sql/alarm/JpaAlarmDao.java | 7 ++ .../server/dao/sql/edge/JpaEdgeDao.java | 5 + .../dao/sql/entityview/JpaEntityViewDao.java | 6 ++ .../dao/sql/resource/JpaTbResourceDao.java | 7 ++ .../server/dao/sql/rpc/JpaRpcDao.java | 7 ++ .../server/dao/sql/rule/JpaRuleNodeDao.java | 7 ++ .../server/dao/sql/tenant/JpaTenantDao.java | 6 ++ .../sql/usagerecord/JpaApiUsageStateDao.java | 7 ++ .../server/dao/sql/user/JpaUserDao.java | 7 ++ .../dao/sql/widget/JpaWidgetTypeDao.java | 7 ++ .../dao/sql/widget/JpaWidgetsBundleDao.java | 7 ++ 25 files changed, 291 insertions(+), 160 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f158ff15e7..f4abd1b5f7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.EntitiesExportImportService; @@ -81,9 +80,10 @@ public class EntitiesExportImportController extends BaseController { public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, @PathVariable UUID id, @RequestParam Map exportSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); try { - return exportEntity(getTenantId(), entityId, toExportSettings(exportSettingsParams)); + return exportEntity(user, entityId, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -93,11 +93,12 @@ public class EntitiesExportImportController extends BaseController { public List>> exportEntitiesByIds(@PathVariable EntityType entityType, @RequestParam UUID[] ids, @RequestParam Map exportSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); List entitiesIds = Arrays.stream(ids) .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) .collect(Collectors.toList()); try { - return exportEntitiesByIds(getTenantId(), entitiesIds, toExportSettings(exportSettingsParams)); + return exportEntitiesByIds(user, entitiesIds, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -109,13 +110,13 @@ public class EntitiesExportImportController extends BaseController { @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "2147483647") int pageSize, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); entityTypeFilter.setEntityType(entityType); try { - return exportEntitiesByFilter(tenantId, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); + return exportEntitiesByFilter(user, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -127,10 +128,10 @@ public class EntitiesExportImportController extends BaseController { @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "2147483647") int pageSize, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { - return exportEntitiesByFilter(tenantId, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); + return exportEntitiesByFilter(user, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -141,12 +142,12 @@ public class EntitiesExportImportController extends BaseController { public List>> exportAllEntitiesByFilters(@RequestBody List filters, @RequestParam Map exportSettingsParams, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { List>> exportDataList = new ArrayList<>(); for (EntityFilter filter : filters) { - exportDataList.addAll(exportEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + exportDataList.addAll(exportEntitiesByFilter(user, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); } return exportDataList; } catch (Exception e) { @@ -158,17 +159,17 @@ public class EntitiesExportImportController extends BaseController { public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, @RequestParam Map exportSettingsParams, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { - return exportEntitiesByQuery(tenantId, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); + return exportEntitiesByQuery(user, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } - private List>> exportEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + private List>> exportEntitiesByFilter(SecurityUser user, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); pageLink.setPageSize(pageSize); @@ -176,43 +177,26 @@ public class EntitiesExportImportController extends BaseController { pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return exportEntitiesByQuery(tenantId, customerId, query, exportSettings); + return exportEntitiesByQuery(user, customerId, query, exportSettings); } - private List>> exportEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { - List entitiesIds = entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + private List>> exportEntitiesByQuery(SecurityUser user, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { + List entitiesIds = entityService.findEntityDataByQuery(user.getTenantId(), customerId, query).getData().stream() .map(EntityData::getEntityId) .collect(Collectors.toList()); - return exportEntitiesByIds(tenantId, entitiesIds, exportSettings); + return exportEntitiesByIds(user, entitiesIds, exportSettings); } - private List>> exportEntitiesByIds(TenantId tenantId, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { + private List>> exportEntitiesByIds(SecurityUser user, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { List>> exportDataList = new ArrayList<>(); for (EntityId entityId : entitiesIds) { - exportDataList.add(exportEntity(tenantId, entityId, exportSettings)); + exportDataList.add(exportEntity(user, entityId, exportSettings)); } return exportDataList; } - private , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - checkEntityId(entityId, Operation.READ); - - List relations = new LinkedList<>(); - if (exportSettings.isExportInboundRelations()) { - relations.addAll(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)); - } - if (exportSettings.isExportOutboundRelations()) { - relations.addAll(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)); - } - for (EntityRelation relation : relations) { - if (!relation.getFrom().equals(entityId)) { - checkEntityId(relation.getFrom(), Operation.READ); - } else if (!relation.getTo().equals(entityId)) { - checkEntityId(relation.getTo(), Operation.READ); - } - } - - return exportImportService.exportEntity(tenantId, entityId, exportSettings); + private , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + return exportImportService.exportEntity(user, entityId, exportSettings); } @@ -235,41 +219,7 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - for (EntityExportData> exportData : exportDataList) { - ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getEntity().getId()); - if (existingEntity != null) { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.WRITE, existingEntity.getId(), existingEntity); - } else { - exportData.getEntity().setTenantId(user.getTenantId()); - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE, null, exportData.getEntity()); - } - - List relations = new LinkedList<>(); - if (importSettings.isImportInboundRelations() && exportData.getInboundRelations() != null) { - relations.addAll(exportData.getInboundRelations()); - } - if (importSettings.isImportOutboundRelations() && exportData.getOutboundRelations() != null) { - relations.addAll(exportData.getOutboundRelations()); - } - for (EntityRelation relation : relations) { - EntityId otherEntityId = null; - if (!relation.getFrom().equals(exportData.getEntity().getId())) { - otherEntityId = relation.getFrom(); - } else if (!relation.getTo().equals(exportData.getEntity().getId())) { - otherEntityId = relation.getTo(); - } - if (otherEntityId != null) { - ExportableEntity otherEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), otherEntityId); - if (otherEntity != null) { - checkEntityId(otherEntity.getId(), Operation.WRITE); - } else { - throw new IllegalArgumentException("Relation is referencing non-existing entity"); - } - } - } - } - - return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + return exportImportService.importEntities(user, exportDataList, importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index ef9da1f89d..7d98ae9004 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -15,19 +15,23 @@ */ package org.thingsboard.server.service.sync; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; -import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; @@ -36,7 +40,10 @@ import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -45,12 +52,15 @@ import java.util.stream.Collectors; @Service @TbCoreComponent +@RequiredArgsConstructor public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); + private final AccessControlService accessControlService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD @@ -58,52 +68,67 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { + public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(tenantId, entityId, exportSettings); + return exportService.getExportData(user, entityId, exportSettings); } // TODO [viacheslav]: validate export data - @Transactional + @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { + public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - return importService.importEntity(tenantId, exportData, importSettings); + return importService.importEntity(user, exportData, importSettings); } - @Transactional + @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { - return exportDataList.stream() - .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) - // TODO [viacheslav]: order for rule chains (depending on references) - .map(exportData -> importEntity(tenantId, exportData, importSettings)) - .collect(Collectors.toList()); + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + + List>> importResults = new ArrayList<>(); + for (EntityExportData> exportData : exportDataList) { + importResults.add(importEntity(user, exportData, importSettings)); + } + return importResults; } @Override - public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + public , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId) { EntityType entityType = externalId.getEntityType(); if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - E entity = dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - if (entity != null) { - return entity; - } + return dao.findByTenantIdAndExternalId(user.getTenantId().getId(), externalId.getId()); } - return findEntityById(tenantId, externalId); + return findEntityById(user, externalId); } @Override - public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + public , I extends EntityId> E findEntityById(SecurityUser user, I id) { Dao dao = (Dao) getDao(id.getEntityType()); - return dao.findById(tenantId, id.getId()); + return dao.findById(user.getTenantId(), id.getId()); + } + + + @Override + public void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException { + if (entity instanceof HasTenantId) { + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), (HasTenantId) entity); + } else if (entity != null) { + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation); + } + } + + @Override + public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { + HasId entity = findEntityById(user, entityId); + checkPermission(user, entity, operation); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 0fbe75a866..447d514c68 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -16,8 +16,9 @@ package org.thingsboard.server.service.sync; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; @@ -27,10 +28,10 @@ import java.util.List; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); + , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); + , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException; - , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings); + List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 6b57cbb557..681a398720 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { - D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings); + D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 9a47e98920..c46ea14b10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,14 +16,22 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId); - , I extends EntityId> E findEntityById(TenantId tenantId, I id); + , I extends EntityId> E findEntityById(SecurityUser user, I id); + + + void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException; + + void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index a3a83196bb..66558634a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -18,14 +18,17 @@ package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.List; @@ -38,19 +41,31 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + if (inboundRelations != null) { + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + } exportData.setInboundRelations(inboundRelations); } if (exportSettings.isExportOutboundRelations()) { - List outboundRelations = relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + if (outboundRelations != null) { + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); + } + } exportData.setOutboundRelations(outboundRelations); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index 19947e8917..e3e40eea2a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.sync.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { - EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); + EntityImportResult importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException; EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index b9013fd549..326e15a91b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -33,9 +33,13 @@ public class AssetImportService extends BaseEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { + public EntityImportResult importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException { E entity = exportData.getEntity(); - E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); + E existingEntity = exportableEntitiesService.findEntityByExternalId(user, entity.getId()); entity.setExternalId(entity.getId()); + NewIdProvider idProvider = new NewIdProvider(user, entity, existingEntity, importSettings); + setOwner(user.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); + exportableEntitiesService.checkPermission(user, entity, Operation.CREATE); } else { entity.setId(existingEntity.getId()); + exportableEntitiesService.checkPermission(user, existingEntity, Operation.WRITE); } - E savedEntity = prepareAndSave(tenantId, entity, exportData, new NewIdProvider(entity, existingEntity, importSettings)); - importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); + E savedEntity = prepareAndSave(user.getTenantId(), entity, exportData, idProvider); + importRelations(user, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -73,57 +80,71 @@ public abstract class BaseEntityImportService newRelations = new LinkedList<>(); if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> { - relation.setTo(savedEntity.getId()); - relation.setFrom(getInternalId(tenantId, relation.getFrom())); - }) + .peek(relation -> relation.setTo(savedEntity.getId())) .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelations() && existingEntity != null) { - relationService.findByTo(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { - relationService.deleteRelation(tenantId, existingRelation); - }); + for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> { - relation.setTo(getInternalId(tenantId, relation.getTo())); - relation.setFrom(savedEntity.getId()); - }) + .peek(relation -> relation.setFrom(savedEntity.getId())) .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelations() && existingEntity != null) { - relationService.findByFrom(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { - relationService.deleteRelation(tenantId, existingRelation); - }); + for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - newRelations.forEach(relation -> { - relationService.saveRelation(tenantId, relation); - }); + for (EntityRelation relation : newRelations) { + HasId otherEntity = null; + if (!relation.getTo().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user, relation.getTo()); + relation.setTo(otherEntity.getId()); + } + if (!relation.getFrom().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user, relation.getFrom()); + relation.setFrom(otherEntity.getId()); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + } + + relationService.saveRelation(user.getTenantId(), relation); + } } - private ID getInternalId(TenantId tenantId, ID externalId) { + private , ID extends EntityId> IE findInternalEntity(SecurityUser user, ID externalId) { if (externalId == null || externalId.isNullUid()) { return null; } - HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); + IE entity = exportableEntitiesService.findEntityByExternalId(user, externalId); if (entity == null) { throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } - return entity.getId(); + return entity; } @RequiredArgsConstructor protected class NewIdProvider { + private final SecurityUser user; private final E entity; private final E existingEntity; private final EntityImportSettings importSettings; @@ -132,26 +153,37 @@ public abstract class BaseEntityImportService ID get(TenantId tenantId, Function idExtractor) { + public ID get(Function idExtractor) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { - return getInternalId(tenantId, idExtractor.apply(entity)); + return getInternalId(idExtractor.apply(this.entity)); } else { return idExtractor.apply(existingEntity); } } - public Set get(TenantId tenantId, Function> listExtractor, Function idGetter, BiConsumer idSetter) { + public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() .peek(t -> { - idSetter.accept(t, getInternalId(tenantId, idGetter.apply(t))); + idSetter.accept(t, getInternalId(idGetter.apply(t))); }) .collect(Collectors.toSet()); } else { return listExtractor.apply(existingEntity); } } + + private ID getInternalId(ID externalId) { + try { + HasId entity = findInternalEntity(user, externalId); + exportableEntitiesService.checkPermission(user, entity, Operation.READ); + return entity.getId(); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 92f7371d50..d17e6c8b41 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -33,8 +33,12 @@ public class CustomerImportService extends BaseEntityImportService assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); + Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); dashboard = dashboardService.saveDashboard(dashboard); for (ShortCustomerInfo customerInfo : assignedCustomers) { @@ -54,7 +58,7 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() + Set newAssignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set toUnassign = new HashSet<>(existingAssignedCustomers); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index f35a20e312..3b844f8657 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -33,12 +33,16 @@ public class DeviceImportService extends BaseEntityImportService { @@ -54,18 +58,19 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.get(tenantId, rc -> new RuleChainId(otherRuleChainUuid)).toString() + idProvider.get(rc -> new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(tenantId, rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(rc -> ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); if (ruleChain.getId() != null) { + // FIXME [viacheslav]: maybe no need to delete ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); } ruleChain = ruleChainService.saveRuleChain(ruleChain); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index cdf04f47c5..8e165898a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -188,4 +189,10 @@ public class JpaAlarmDao extends JpaAbstractDao implements A log.trace("[{}] Try to delete entity alarm records using [{}]", tenantId, entityId); entityAlarmRepository.deleteByEntityId(entityId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 4be9c08849..802c9b99b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -193,4 +193,9 @@ public class JpaEdgeDao extends JpaAbstractSearchTextDao imple return list; } + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 8c515f13c0..52371706e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -200,4 +200,10 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao implements RpcDao public Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + + @Override + public EntityType getEntityType() { + return EntityType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index a499e68da8..d49b54434f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; @@ -50,4 +51,10 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search) { return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByTenantIdAndType(tenantId.getId(), type, search)); } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index b985d45e8c..56a75f4d3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.id.TenantId; @@ -78,4 +79,9 @@ public class JpaTenantDao extends JpaAbstractSearchTextDao return DaoUtil.pageToPageData(tenantRepository.findTenantsIds(DaoUtil.toPageable(pageLink))).mapData(TenantId::fromUUID); } + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 7c6801042d..96a0ecaede 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.usagerecord; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -68,4 +69,10 @@ public class JpaApiUsageStateDao extends JpaAbstractDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index bbaf2529a7..eac0a15b1e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; @@ -73,4 +74,10 @@ public class JpaWidgetTypeDao extends JpaAbstractDao Date: Tue, 5 Apr 2022 17:12:51 +0300 Subject: [PATCH 035/178] Add option to find existing entity by name when importing; refactoring --- .../EntitiesExportImportController.java | 35 ++++++----- .../DefaultEntitiesExportImportService.java | 35 ++++++----- .../exporting/ExportableEntitiesService.java | 11 ++-- .../impl/BaseEntityExportService.java | 7 ++- .../sync/importing/EntityImportSettings.java | 1 + .../impl/BaseEntityImportService.java | 58 ++++++++++++------- .../server/dao/ExportableEntityDao.java | 2 + .../server/dao/asset/BaseAssetService.java | 2 + .../server/dao/sql/asset/JpaAssetDao.java | 5 ++ .../dao/sql/customer/JpaCustomerDao.java | 5 ++ .../sql/dashboard/DashboardRepository.java | 2 + .../dao/sql/dashboard/JpaDashboardDao.java | 5 ++ .../server/dao/sql/device/JpaDeviceDao.java | 5 ++ .../dao/sql/device/JpaDeviceProfileDao.java | 5 ++ .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ++ .../dao/sql/rule/RuleChainRepository.java | 2 + 16 files changed, 127 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f4abd1b5f7..25d48c54a7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -38,23 +37,18 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; -import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -72,7 +66,6 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final ExportableEntitiesService exportableEntitiesService; private final EntityService entityService; @@ -201,8 +194,8 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") - public List>> importEntity(@RequestBody List>> exportDataList, - @RequestParam Map importSettingsParams) throws ThingsboardException { + public List>> importEntities(@RequestBody List>> exportDataList, + @RequestParam Map importSettingsParams) throws ThingsboardException { SecurityUser user = getCurrentUser(); EntityImportSettings importSettings = toImportSettings(importSettingsParams); @@ -225,25 +218,31 @@ public class EntitiesExportImportController extends BaseController { private EntityExportSettings toExportSettings(Map exportSettingsParams) { return EntityExportSettings.builder() - .exportInboundRelations(getParam(exportSettingsParams, "exportInboundRelations", false, Boolean::parseBoolean)) - .exportOutboundRelations(getParam(exportSettingsParams, "exportOutboundRelations", false, Boolean::parseBoolean)) + .exportInboundRelations(getBooleanParam(exportSettingsParams, "exportInboundRelations", false)) + .exportOutboundRelations(getBooleanParam(exportSettingsParams, "exportOutboundRelations", false)) .build(); } private EntityImportSettings toImportSettings(Map importSettingsParams) { return EntityImportSettings.builder() - .importInboundRelations(getParam(importSettingsParams, "importInboundRelations", false, Boolean::parseBoolean)) - .importOutboundRelations(getParam(importSettingsParams, "importOutboundRelations", false, Boolean::parseBoolean)) - .removeExistingRelations(getParam(importSettingsParams, "removeExistingRelations", true, Boolean::parseBoolean)) - .updateReferencesToOtherEntities(getParam(importSettingsParams, "updateReferencesToOtherEntities", true, Boolean::parseBoolean)) + .findExistingByName(getBooleanParam(importSettingsParams, "findExistingByName", false)) + .importInboundRelations(getBooleanParam(importSettingsParams, "importInboundRelations", false)) + .importOutboundRelations(getBooleanParam(importSettingsParams, "importOutboundRelations", false)) + .removeExistingRelations(getBooleanParam(importSettingsParams, "removeExistingRelations", true)) + .updateReferencesToOtherEntities(getBooleanParam(importSettingsParams, "updateReferencesToOtherEntities", true)) .build(); } - protected T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { + + protected static boolean getBooleanParam(Map requestParams, String key, boolean defaultValue) { + return getParam(requestParams, key, defaultValue, Boolean::parseBoolean); + } + + protected static T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); } - private CustomerId toCustomerId(UUID customerUuid) { + private static CustomerId toCustomerId(UUID customerUuid) { return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 7d98ae9004..8bfaa9fe80 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -41,14 +42,11 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Service @TbCoreComponent @@ -100,35 +98,46 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId) { + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { EntityType entityType = externalId.getEntityType(); if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findByTenantIdAndExternalId(user.getTenantId().getId(), externalId.getId()); + return dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } else { + return null; } - return findEntityById(user, externalId); } @Override - public , I extends EntityId> E findEntityById(SecurityUser user, I id) { + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { Dao dao = (Dao) getDao(id.getEntityType()); - return dao.findById(user.getTenantId(), id.getId()); + E entity = dao.findById(tenantId, id.getId()); + if (entity instanceof HasTenantId && !((HasTenantId) entity).getTenantId().equals(tenantId)) { + return null; + } + return entity; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); + return dao.findFirstByTenantIdAndName(tenantId.getId(), name); } @Override - public void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException { + public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { if (entity instanceof HasTenantId) { - accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), (HasTenantId) entity); + accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); } else if (entity != null) { - accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation); + accessControlService.checkPermission(user, Resource.of(entityType), operation); } } @Override public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { - HasId entity = findEntityById(user, entityId); - checkPermission(user, entity, operation); + HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); + checkPermission(user, entity, entityId.getEntityType(), operation); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index c46ea14b10..5cf31042e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -15,22 +15,25 @@ */ package org.thingsboard.server.service.sync.exporting; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId); + , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId); - , I extends EntityId> E findEntityById(SecurityUser user, I id); + , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id); + , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); - void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException; + + void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 66558634a4..f098485fd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -44,8 +44,11 @@ public abstract class BaseEntityExportService importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException { E entity = exportData.getEntity(); - E existingEntity = exportableEntitiesService.findEntityByExternalId(user, entity.getId()); + E existingEntity = findExistingEntity(user.getTenantId(), entity, importSettings); entity.setExternalId(entity.getId()); @@ -65,10 +65,10 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) + .or(() -> { + if (importSettings.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); + } + + private void importRelations(SecurityUser user, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) throws ThingsboardException { List newRelations = new LinkedList<>(); @@ -116,31 +138,21 @@ public abstract class BaseEntityImportService otherEntity = null; if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user, relation.getTo()); + otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); relation.setTo(otherEntity.getId()); } if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user, relation.getFrom()); + otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); relation.setFrom(otherEntity.getId()); } if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); } relationService.saveRelation(user.getTenantId(), relation); } } - private , ID extends EntityId> IE findInternalEntity(SecurityUser user, ID externalId) { - if (externalId == null || externalId.isNullUid()) { - return null; - } - IE entity = exportableEntitiesService.findEntityByExternalId(user, externalId); - if (entity == null) { - throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); - } - return entity; - } @RequiredArgsConstructor protected class NewIdProvider { @@ -175,12 +187,16 @@ public abstract class BaseEntityImportService ID getInternalId(ID externalId) { - try { - HasId entity = findInternalEntity(user, externalId); - exportableEntitiesService.checkPermission(user, entity, Operation.READ); + HasId entity = findInternalEntity(user.getTenantId(), externalId); + if (entity != null) { + try { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } return entity.getId(); - } catch (ThingsboardException e) { - throw new IllegalArgumentException(e.getMessage(), e); + } else { + return null; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 7e4d00286c..467389d9d0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -23,4 +23,6 @@ public interface ExportableEntityDao> extends Dao< T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + T findFirstByTenantIdAndName(UUID tenantId, String name); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index bf8d75d27a..4f31f3e8c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -110,6 +111,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ .orElse(null); } + @Transactional @CacheEvict(cacheNames = ASSET_CACHE, key = "{#asset.tenantId, #asset.name}") @Override public Asset saveAsset(Asset asset) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 14788f8b38..5988c2d2db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -214,6 +214,11 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); } + @Override + public Asset findFirstByTenantIdAndName(UUID tenantId, String name) { + return findAssetsByTenantIdAndName(tenantId, name).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 17e1625a29..fa09304427 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -75,6 +75,11 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); } + @Override + public Device findFirstByTenantIdAndName(UUID tenantId, String name) { + return findDeviceByTenantIdAndName(tenantId, name).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 23e564b735..b21a07442f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,6 +116,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + RuleChainEntity findFirstByTenantIdAndName(UUID tenantId, String name); + } From cecf49a67dc60119ad9cd5b65fd49240d5ef5447 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 5 Apr 2022 18:08:26 +0300 Subject: [PATCH 036/178] Fix dashboard import to update entity aliases --- .../DefaultEntitiesExportImportService.java | 7 +++- .../impl/DashboardImportService.java | 33 ++++++++++++++++++- .../importing/impl/DeviceImportService.java | 2 +- .../server/dao/asset/BaseAssetService.java | 2 -- .../query/DefaultEntityQueryRepository.java | 2 +- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 8bfaa9fe80..90f6f1c778 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -74,13 +75,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - // TODO [viacheslav]: validate export data @Transactional(rollbackFor = Exception.class) @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); + // TODO [viacheslav]: might throw DataIntegrityViolationException with cause of ConstraintViolationException: need to give normal error return importService.importEntity(user, exportData, importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 7d0ff17ead..67f4ffc4a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -15,15 +15,22 @@ */ package org.thingsboard.server.service.sync.importing.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; @@ -31,6 +38,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Service @@ -40,14 +49,36 @@ public class DashboardImportService extends BaseEntityImportService Optional.ofNullable(configuration.get("entityAliases"))) + .filter(JsonNode::isObject) + .ifPresent(entityAliases -> entityAliases.forEach(entityAlias -> { + Optional.ofNullable(entityAlias.get("filter")) + .filter(JsonNode::isObject) + .ifPresent(filter -> { + EntityFilter entityFilter = JacksonUtil.treeToValue(filter, EntityFilter.class); + EntityType entityType = DefaultEntityQueryRepository.resolveEntityType(entityFilter); + + String filterJson = filter.toString(); + String newFilterJson = UUID_PATTERN.matcher(filterJson).replaceAll(matchResult -> { + String uuid = matchResult.group(); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); + return idProvider.get(d -> entityId).toString(); + }); + ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); + }); + })); + + // TODO [viacheslav]: improve the code below if (dashboard.getId() == null) { Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 3b844f8657..ccff72dcdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -47,7 +47,7 @@ public class DeviceImportService extends BaseEntityImportService Date: Wed, 6 Apr 2022 00:29:22 +0300 Subject: [PATCH 037/178] Refactoring --- .../server/controller/AssetController.java | 6 +- .../server/controller/BaseController.java | 66 ------------------- .../server/controller/CustomerController.java | 10 ++- .../controller/DashboardController.java | 4 +- .../server/controller/DeviceController.java | 14 ++-- .../controller/DeviceProfileController.java | 25 ++++++- .../EntitiesExportImportController.java | 12 ++-- .../controller/RuleChainController.java | 6 +- .../service/action/EntityActionService.java | 59 ++++++++++++++++- .../impl/BaseEntityExportService.java | 15 +++-- .../sync/importing/EntityImportResult.java | 5 ++ .../importing/impl/AssetImportService.java | 9 +++ .../impl/BaseEntityImportService.java | 65 ++++++++++-------- .../importing/impl/CustomerImportService.java | 11 ++++ .../impl/DashboardImportService.java | 11 ++++ .../importing/impl/DeviceImportService.java | 13 ++++ .../impl/DeviceProfileImportService.java | 9 +++ .../impl/RuleChainImportService.java | 9 +++ .../server/utils/ThrowingRunnable.java | 24 +++++++ 19 files changed, 245 insertions(+), 128 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index b05e32efe7..5e44e1433a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -155,9 +155,7 @@ public class AssetController extends BaseController { checkEntity(asset.getId(), asset, Resource.ASSET); Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedAsset, null, asset.getId() == null); - + entityActionService.onAssetCreatedOrUpdated(getCurrentUser(), savedAsset, asset.getId() == null); return savedAsset; } catch (Exception e) { logEntityAction(emptyId(EntityType.ASSET), asset, @@ -667,7 +665,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - onEntityUpdatedOrCreated(user, importedAssetInfo.getEntity(), importedAssetInfo.getOldEntity(), !importedAssetInfo.isUpdated()); + entityActionService.onAssetCreatedOrUpdated(user, importedAssetInfo.getEntity(), !importedAssetInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 1f683c7874..f945cc30b7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.OtaPackage; @@ -69,7 +68,6 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -85,7 +83,6 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rpc.Rpc; @@ -144,7 +141,6 @@ import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -923,66 +919,4 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } - - protected , I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { - boolean notifyEdgeService = false; - - EntityType entityType = savedEntity.getId().getEntityType(); - switch (entityType) { - case DEVICE: - tbClusterService.onDeviceUpdated((Device) savedEntity, (Device) oldEntity); - break; - case DEVICE_PROFILE: - DeviceProfile deviceProfile = (DeviceProfile) savedEntity; - DeviceProfile oldDeviceProfile = (DeviceProfile) oldEntity; - boolean isFirmwareChanged = false; - boolean isSoftwareChanged = false; - if (!isNewEntity) { - if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { - isFirmwareChanged = true; - } - if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { - isSoftwareChanged = true; - } - } - tbClusterService.onDeviceProfileChange(deviceProfile, null); - tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), - isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); - notifyEdgeService = true; - break; - case RULE_CHAIN: - RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); - if (RuleChainType.CORE.equals(ruleChainType)) { - tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedEntity.getId(), - isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - if (RuleChainType.EDGE.equals(ruleChainType)) { - if (!isNewEntity) { - notifyEdgeService = true; - } - } - break; - case ASSET: - case CUSTOMER: - case DASHBOARD: - if (!isNewEntity) { - notifyEdgeService = true; - } - break; - default: - throw new UnsupportedOperationException(); - } - - try { - logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, - isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - if (notifyEdgeService) { - sendEntityNotificationMsg(user.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 27037f36e7..b419121459 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -33,6 +33,7 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; @@ -150,7 +151,14 @@ public class CustomerController extends BaseController { checkEntity(customer.getId(), customer, Resource.CUSTOMER); Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - onEntityUpdatedOrCreated(getCurrentUser(), savedCustomer, null, customer.getId() == null); + + logEntityAction(savedCustomer.getId(), savedCustomer, + savedCustomer.getId(), + customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + if (customer.getId() != null) { + sendEntityNotificationMsg(savedCustomer.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 4ea1ca83c7..b4b6d6be89 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -186,9 +186,7 @@ public class DashboardController extends BaseController { checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD); Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDashboard, null, dashboard.getId() == null); - + entityActionService.onDashboardCreatedOrUpdated(getCurrentUser(), savedDashboard, dashboard.getId() == null); return savedDashboard; } catch (Exception e) { logEntityAction(emptyId(EntityType.DASHBOARD), dashboard, diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index feb58edb7a..2dff62aec6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,11 +74,11 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import javax.annotation.Nullable; import java.io.IOException; @@ -194,9 +194,7 @@ public class DeviceController extends BaseController { } Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, oldDevice, created); - + entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, oldDevice, created); return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -224,9 +222,7 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, device, device.getId() == null); - + entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, device, created); return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -1001,7 +997,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - onEntityUpdatedOrCreated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); + entityActionService.onDeviceCreatedOrUpdated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index 240e60be2c..25a35e77fa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -46,6 +46,7 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; +import java.util.Objects; import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_DATA; @@ -206,14 +207,32 @@ public class DeviceProfileController extends BaseController { checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); - DeviceProfile oldDeviceProfile = null; + boolean isFirmwareChanged = false; + boolean isSoftwareChanged = false; + if (!created) { - oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); + DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); + if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { + isFirmwareChanged = true; + } + if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { + isSoftwareChanged = true; + } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - onEntityUpdatedOrCreated(getCurrentUser(), deviceProfile, oldDeviceProfile, created); + tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); + tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), savedDeviceProfile.getId(), + created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + + logEntityAction(savedDeviceProfile.getId(), savedDeviceProfile, + null, + created ? ActionType.ADDED : ActionType.UPDATED, null); + + otaPackageStateService.update(savedDeviceProfile, isFirmwareChanged, isSoftwareChanged); + sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(), + deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); return savedDeviceProfile; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile, diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 25d48c54a7..6d6a2e4619 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -200,11 +200,13 @@ public class EntitiesExportImportController extends BaseController { EntityImportSettings importSettings = toImportSettings(importSettingsParams); try { - return importEntities(user, exportDataList, importSettings) - .stream().peek(entityImportResult -> { - onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); - }) - .collect(Collectors.toList()); + List>> importResults = importEntities(user, exportDataList, importSettings); + for (EntityImportResult> entityImportResult : importResults) { + if (entityImportResult.getCallback() != null) { + entityImportResult.getCallback().run(); + } + } + return importResults; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index cb59951ac5..e27090c700 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -78,9 +78,11 @@ import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -247,12 +249,10 @@ public class RuleChainController extends BaseController { try { boolean created = ruleChain.getId() == null; ruleChain.setTenantId(getCurrentUser().getTenantId()); - checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedRuleChain, null, created); + entityActionService.onRuleChainCreatedOrUpdated(getCurrentUser(), savedRuleChain, created); return savedRuleChain; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index b8c0656bd2..d692e24bbe 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,12 +22,17 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -36,13 +41,20 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -212,7 +224,7 @@ public class EntityActionService { } } - public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, + public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, ActionType actionType, Exception e, Object... additionalInfo) { if (customerId == null || customerId.isNullUid()) { customerId = user.getCustomerId(); @@ -267,4 +279,49 @@ public class EntityActionService { entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString()); } } + + + public void onDeviceCreatedOrUpdated(SecurityUser user, Device savedDevice, Device oldDevice, boolean newEntity) { + tbClusterService.onDeviceUpdated(savedDevice, oldDevice); + logEntityAction(user, savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + } + + public void onAssetCreatedOrUpdated(SecurityUser user, Asset savedAsset, boolean newEntity) { + logEntityAction(user, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedAsset.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + + public void onDashboardCreatedOrUpdated(SecurityUser user, Dashboard savedDashboard, boolean newEntity) { + logEntityAction(user, savedDashboard.getId(), savedDashboard, null, + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedDashboard.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + + public void onRuleChainCreatedOrUpdated(SecurityUser user, RuleChain savedRuleChain, boolean newEntity) { + if (RuleChainType.CORE.equals(savedRuleChain.getType())) { + tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + newEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + logEntityAction(user, savedRuleChain.getId(), savedRuleChain, null, + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedRuleChain.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + } + + public void onCustomerCreatedOrUpdated(SecurityUser user, Customer savedCustomer) { + + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index f098485fd6..772e010b2b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -52,9 +52,16 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); if (inboundRelations != null) { for (EntityRelation relation : inboundRelations) { exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); @@ -63,7 +70,7 @@ public abstract class BaseEntityExportService outboundRelations = relationService.findByFrom(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); if (outboundRelations != null) { for (EntityRelation relation : outboundRelations) { exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); @@ -71,12 +78,8 @@ public abstract class BaseEntityExportService> { private E savedEntity; private E oldEntity; + @JsonIgnore + private transient ThrowingRunnable callback; // to call when entity is successfully saved and transaction is committed } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 326e15a91b..1c031f50e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -23,7 +23,9 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.AssetExportData; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -43,6 +45,13 @@ public class AssetImportService extends BaseEntityImportService { + entityActionService.onAssetCreatedOrUpdated(user, savedAsset, oldAsset == null); + }; + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 42e9263e77..bf3c126c92 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; @@ -36,6 +37,7 @@ import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.LinkedList; @@ -52,6 +54,8 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); + importResult.setCallback(callback); return importResult; } @@ -84,30 +89,8 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) - .or(() -> { - if (importSettings.isFindExistingByName()) { - return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); - } else { - return Optional.empty(); - } - }) - .orElse(null); - } - - private HasId findInternalEntity(TenantId tenantId, ID externalId) { - if (externalId == null || externalId.isNullUid()) return null; - - return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) - .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) - .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); - } - - - private void importRelations(SecurityUser user, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) throws ThingsboardException { + protected ThrowingRunnable processAfterSavedAndGetCallback(SecurityUser user, E savedEntity, E oldEntity, D exportData, + EntityImportSettings importSettings, NewIdProvider idProvider) throws ThingsboardException { List newRelations = new LinkedList<>(); if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { @@ -115,7 +98,7 @@ public abstract class BaseEntityImportService relation.setTo(savedEntity.getId())) .collect(Collectors.toList())); - if (importSettings.isRemoveExistingRelations() && existingEntity != null) { + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); relationService.deleteRelation(user.getTenantId(), existingRelation); @@ -127,7 +110,7 @@ public abstract class BaseEntityImportService relation.setFrom(savedEntity.getId())) .collect(Collectors.toList())); - if (importSettings.isRemoveExistingRelations() && existingEntity != null) { + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); relationService.deleteRelation(user.getTenantId(), existingRelation); @@ -151,6 +134,34 @@ public abstract class BaseEntityImportService {}; + } + + + private E findExistingEntity(TenantId tenantId, E entity, EntityImportSettings importSettings) { + return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, entity.getId())) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) + .or(() -> { + if (importSettings.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index d17e6c8b41..336d9f68f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -19,11 +19,15 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -42,6 +46,13 @@ public class CustomerImportService extends BaseEntityImportService { + entityActionService.onCustomerCreatedOrUpdated(user, savedCustomer); + }; + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 67f4ffc4a7..af24576d34 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -32,7 +33,10 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; @@ -108,6 +112,13 @@ public class DashboardImportService extends BaseEntityImportService { + entityActionService.onDashboardCreatedOrUpdated(user, savedDashboard, oldDashboard == null); + }; + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index ccff72dcdb..4212083dea 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -17,13 +17,18 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -31,6 +36,7 @@ import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; + private final TbClusterService clusterService; @Override protected void setOwner(TenantId tenantId, Device device, NewIdProvider idProvider) { @@ -52,6 +58,13 @@ public class DeviceImportService extends BaseEntityImportService { + clusterService.onDeviceUpdated(savedDevice, oldDevice); + }; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index b59d4fa23e..f9d8d2ddd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -23,7 +23,9 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -46,6 +48,13 @@ public class DeviceProfileImportService extends BaseEntityImportService { + + }; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 8f0ff5dc7e..f6a96f24bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -27,7 +27,9 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.Optional; @@ -79,6 +81,13 @@ public class RuleChainImportService extends BaseEntityImportService { + + }; + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java new file mode 100644 index 0000000000..6965a79d19 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import org.thingsboard.server.common.data.exception.ThingsboardException; + +public interface ThrowingRunnable { + + void run() throws ThingsboardException; + +} From 12840c81e997ea0084292f608e9d7b522e786ece Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 6 Apr 2022 21:39:26 +0300 Subject: [PATCH 038/178] Refactor events sending on import; export api with ExportRequest of different types --- .../server/controller/AssetController.java | 20 +- .../controller/DashboardController.java | 10 +- .../server/controller/DeviceController.java | 28 ++- .../EntitiesExportImportController.java | 199 ++++++------------ .../controller/RuleChainController.java | 17 +- .../service/action/EntityActionService.java | 61 +----- .../DefaultEntitiesExportImportService.java | 2 +- .../sync/EntitiesExportImportService.java | 2 +- .../sync/exporting/EntityExportService.java | 1 + .../request}/EntityExportSettings.java | 8 +- .../request/EntityFilterExportRequest.java | 37 ++++ .../data/request/EntityListExportRequest.java | 35 +++ .../request/EntityQueryExportRequest.java | 35 +++ .../data/request/EntityTypeExportRequest.java | 37 ++++ .../exporting/data/request/ExportRequest.java | 41 ++++ .../data/request/ExportRequestType.java | 25 +++ .../request/SingleEntityExportRequest.java | 33 +++ .../impl/BaseEntityExportService.java | 2 +- .../importing/data/request/ImportRequest.java | 32 +++ .../importing/impl/AssetImportService.java | 9 +- .../impl/BaseEntityImportService.java | 10 +- .../importing/impl/CustomerImportService.java | 9 +- .../impl/DashboardImportService.java | 12 +- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 19 +- .../impl/RuleChainImportService.java | 14 +- .../server/utils/ThrowingRunnable.java | 7 + ...EntitiesExportImportControllerSqlTest.java | 2 + 28 files changed, 479 insertions(+), 236 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/{ => data/request}/EntityExportSettings.java (79%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 5e44e1433a..e48f2dcf23 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -155,7 +155,9 @@ public class AssetController extends BaseController { checkEntity(asset.getId(), asset, Resource.ASSET); Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - entityActionService.onAssetCreatedOrUpdated(getCurrentUser(), savedAsset, asset.getId() == null); + + onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser()); + return savedAsset; } catch (Exception e) { logEntityAction(emptyId(EntityType.ASSET), asset, @@ -164,6 +166,20 @@ public class AssetController extends BaseController { } } + private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) { + try { + logEntityAction(user, asset.getId(), asset, + asset.getCustomerId(), + updated ? ActionType.UPDATED : ActionType.ADDED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } + + if (updated) { + sendEntityNotificationMsg(asset.getTenantId(), asset.getId(), EdgeEventActionType.UPDATED); + } + } + @ApiOperation(value = "Delete asset (deleteAsset)", notes = "Deletes the asset and all the relations (from and to the asset). Referencing non-existing asset Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -665,7 +681,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - entityActionService.onAssetCreatedOrUpdated(user, importedAssetInfo.getEntity(), !importedAssetInfo.isUpdated()); + onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index b4b6d6be89..49c33045a5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -186,7 +186,15 @@ public class DashboardController extends BaseController { checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD); Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - entityActionService.onDashboardCreatedOrUpdated(getCurrentUser(), savedDashboard, dashboard.getId() == null); + + logEntityAction(savedDashboard.getId(), savedDashboard, + null, + dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + if (dashboard.getId() != null) { + sendEntityNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + return savedDashboard; } catch (Exception e) { logEntityAction(emptyId(EntityType.DASHBOARD), dashboard, diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 2dff62aec6..03f0996ea9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,11 +74,11 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import javax.annotation.Nullable; import java.io.IOException; @@ -194,7 +194,9 @@ public class DeviceController extends BaseController { } Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, oldDevice, created); + + onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser()); + return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -222,7 +224,11 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, device, created); + tbClusterService.onDeviceUpdated(savedDevice, device); + logEntityAction(savedDevice.getId(), savedDevice, + savedDevice.getCustomerId(), + device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -231,6 +237,18 @@ public class DeviceController extends BaseController { } } + private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) { + tbClusterService.onDeviceUpdated(savedDevice, oldDevice); + + try { + logEntityAction(user, savedDevice.getId(), savedDevice, + savedDevice.getCustomerId(), + updated ? ActionType.UPDATED : ActionType.ADDED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } + } + @ApiOperation(value = "Delete device (deleteDevice)", notes = "Deletes the device, it's credentials and all the relations (from and to the device). Referencing non-existing device Id will cause an error." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -997,7 +1015,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - entityActionService.onDeviceCreatedOrUpdated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); + onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 6d6a2e4619..366eaec41a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,18 +17,16 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -41,13 +39,17 @@ import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -60,7 +62,6 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @RestController @RequestMapping("/api/entities") -@PreAuthorize("hasAuthority('TENANT_ADMIN')") @TbCoreComponent @RequiredArgsConstructor public class EntitiesExportImportController extends BaseController { @@ -68,101 +69,79 @@ public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; private final EntityService entityService; - - @PostMapping("/export/{entityType}/{id}") - public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, - @PathVariable UUID id, - @RequestParam Map exportSettingsParams) throws ThingsboardException { + @PostMapping("/export") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); try { - return exportEntity(user, entityId, toExportSettings(exportSettingsParams)); + return exportEntitiesByRequest(user, exportRequest); } catch (Exception e) { throw handleException(e); } } - @PostMapping(value = "/export/{entityType}", params = "ids") - public List>> exportEntitiesByIds(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam Map exportSettingsParams) throws ThingsboardException { + @PostMapping(value = "/export", params = {"multiple"}) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); try { - return exportEntitiesByIds(user, entitiesIds, toExportSettings(exportSettingsParams)); + List>> exportDataList = new ArrayList<>(); + for (ExportRequest exportRequest : exportRequests) { + exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); + } + return exportDataList; } catch (Exception e) { throw handleException(e); } } - @PostMapping(value = "/export/{entityType}") - public List>> exportEntitiesByEntityType(@PathVariable EntityType entityType, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - try { - return exportEntitiesByFilter(user, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); - } - } + private List>> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { + List entitiesIds = findEntitiesForRequest(user, request); - @PostMapping("/exportByFilter") - public List>> exportEntitiesByFilter(@RequestBody EntityFilter filter, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - return exportEntitiesByFilter(user, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); + List>> exportDataList = new ArrayList<>(); + for (EntityId entityId : entitiesIds) { + exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); } + return exportDataList; } - // FIXME: too aggressive - @PostMapping("/exportByFilters") - public List>> exportAllEntitiesByFilters(@RequestBody List filters, - @RequestParam Map exportSettingsParams, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - List>> exportDataList = new ArrayList<>(); - for (EntityFilter filter : filters) { - exportDataList.addAll(exportEntitiesByFilter(user, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + private List findEntitiesForRequest(SecurityUser user, ExportRequest request) { + switch (request.getType()) { + case SINGLE_ENTITY: { + return List.of(((SingleEntityExportRequest) request).getEntityId()); } - return exportDataList; - } catch (Exception e) { - throw handleException(e); - } - } + case ENTITY_LIST: { + return ((EntityListExportRequest) request).getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(exportRequest.getEntityType()); - @PostMapping("/exportByQuery") - public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, - @RequestParam Map exportSettingsParams, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - return exportEntitiesByQuery(user, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case ENTITY_FILTER: { + EntityFilterExportRequest exportRequest = (EntityFilterExportRequest) request; + EntityFilter filter = exportRequest.getFilter(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case ENTITY_QUERY:{ + EntityQueryExportRequest exportRequest = (EntityQueryExportRequest) request; + EntityDataQuery query = exportRequest.getQuery(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByQuery(user.getTenantId(), customerId, query); + } + default: + throw new IllegalArgumentException("Export request is not supported"); } } - - private List>> exportEntitiesByFilter(SecurityUser user, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); pageLink.setPageSize(pageSize); @@ -170,37 +149,23 @@ public class EntitiesExportImportController extends BaseController { pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return exportEntitiesByQuery(user, customerId, query, exportSettings); + return findEntitiesByQuery(tenantId, customerId, query); } - private List>> exportEntitiesByQuery(SecurityUser user, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { - List entitiesIds = entityService.findEntityDataByQuery(user.getTenantId(), customerId, query).getData().stream() + private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() .map(EntityData::getEntityId) .collect(Collectors.toList()); - return exportEntitiesByIds(user, entitiesIds, exportSettings); - } - - private List>> exportEntitiesByIds(SecurityUser user, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { - List>> exportDataList = new ArrayList<>(); - for (EntityId entityId : entitiesIds) { - exportDataList.add(exportEntity(user, entityId, exportSettings)); - } - return exportDataList; - } - - private , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - return exportImportService.exportEntity(user, entityId, exportSettings); } @PostMapping("/import") - public List>> importEntities(@RequestBody List>> exportDataList, - @RequestParam Map importSettingsParams) throws ThingsboardException { + public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - EntityImportSettings importSettings = toImportSettings(importSettingsParams); - try { - List>> importResults = importEntities(user, exportDataList, importSettings); + List>> importResults = exportImportService.importEntities(user, + importRequest.getExportDataList(), importRequest.getImportSettings()); + for (EntityImportResult> entityImportResult : importResults) { if (entityImportResult.getCallback() != null) { entityImportResult.getCallback().run(); @@ -212,40 +177,4 @@ public class EntitiesExportImportController extends BaseController { } } - - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - return exportImportService.importEntities(user, exportDataList, importSettings); - } - - - private EntityExportSettings toExportSettings(Map exportSettingsParams) { - return EntityExportSettings.builder() - .exportInboundRelations(getBooleanParam(exportSettingsParams, "exportInboundRelations", false)) - .exportOutboundRelations(getBooleanParam(exportSettingsParams, "exportOutboundRelations", false)) - .build(); - } - - private EntityImportSettings toImportSettings(Map importSettingsParams) { - return EntityImportSettings.builder() - .findExistingByName(getBooleanParam(importSettingsParams, "findExistingByName", false)) - .importInboundRelations(getBooleanParam(importSettingsParams, "importInboundRelations", false)) - .importOutboundRelations(getBooleanParam(importSettingsParams, "importOutboundRelations", false)) - .removeExistingRelations(getBooleanParam(importSettingsParams, "removeExistingRelations", true)) - .updateReferencesToOtherEntities(getBooleanParam(importSettingsParams, "updateReferencesToOtherEntities", true)) - .build(); - } - - - protected static boolean getBooleanParam(Map requestParams, String key, boolean defaultValue) { - return getParam(requestParams, key, defaultValue, Boolean::parseBoolean); - } - - protected static T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { - return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); - } - - private static CustomerId toCustomerId(UUID customerUuid) { - return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); - } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index e27090c700..6479b838b2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -249,10 +249,25 @@ public class RuleChainController extends BaseController { try { boolean created = ruleChain.getId() == null; ruleChain.setTenantId(getCurrentUser().getTenantId()); + checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - entityActionService.onRuleChainCreatedOrUpdated(getCurrentUser(), savedRuleChain, created); + + if (RuleChainType.CORE.equals(savedRuleChain.getType())) { + tbClusterService.broadcastEntityStateChangeEvent(ruleChain.getTenantId(), savedRuleChain.getId(), + created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + logEntityAction(savedRuleChain.getId(), savedRuleChain, + null, + created ? ActionType.ADDED : ActionType.UPDATED, null); + + if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { + if (!created) { + sendEntityNotificationMsg(savedRuleChain.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + } return savedRuleChain; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index d692e24bbe..2a64db7385 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,18 +22,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -41,20 +37,12 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainType; -import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -235,6 +223,9 @@ public class EntityActionService { auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo); } + public void sendEntityNotificationMsgToEdgeService(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action); + } private T extractParameter(Class clazz, int index, Object... additionalInfo) { T result = null; @@ -280,48 +271,4 @@ public class EntityActionService { } } - - public void onDeviceCreatedOrUpdated(SecurityUser user, Device savedDevice, Device oldDevice, boolean newEntity) { - tbClusterService.onDeviceUpdated(savedDevice, oldDevice); - logEntityAction(user, savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - } - - public void onAssetCreatedOrUpdated(SecurityUser user, Asset savedAsset, boolean newEntity) { - logEntityAction(user, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedAsset.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - - public void onDashboardCreatedOrUpdated(SecurityUser user, Dashboard savedDashboard, boolean newEntity) { - logEntityAction(user, savedDashboard.getId(), savedDashboard, null, - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedDashboard.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - - public void onRuleChainCreatedOrUpdated(SecurityUser user, RuleChain savedRuleChain, boolean newEntity) { - if (RuleChainType.CORE.equals(savedRuleChain.getType())) { - tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), - newEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - logEntityAction(user, savedRuleChain.getId(), savedRuleChain, null, - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedRuleChain.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - } - - public void onCustomerCreatedOrUpdated(SecurityUser user, Customer savedCustomer) { - - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 90f6f1c778..9bca5f8bc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 447d514c68..3fa9b13406 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 681a398720..c09cbcd36c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index 36552b2c12..01e4f8d068 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -13,17 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exporting.data.request; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder public class EntityExportSettings { private boolean exportInboundRelations; private boolean exportOutboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java new file mode 100644 index 0000000000..00ef0604e5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.EntityFilter; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityFilterExportRequest extends ExportRequest { + + private EntityFilter filter; + private int page; + private int pageSize; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_FILTER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java new file mode 100644 index 0000000000..01309a421b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityListExportRequest extends ExportRequest { + + private List entitiesIds; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_LIST; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java new file mode 100644 index 0000000000..85232821d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityQueryExportRequest extends ExportRequest { + + private EntityDataQuery query; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_QUERY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java new file mode 100644 index 0000000000..82cb023258 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityTypeExportRequest extends ExportRequest { + + private EntityType entityType; + private int page; + private int pageSize; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java new file mode 100644 index 0000000000..c140786d06 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import lombok.Data; + +@JsonTypeInfo(use = Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), + @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), + @Type(name = "ENTITY_FILTER", value = EntityFilterExportRequest.class), + @Type(name = "ENTITY_QUERY", value = EntityQueryExportRequest.class) +}) +@Data +public abstract class ExportRequest { + + private EntityExportSettings exportSettings; + + @JsonIgnore + public abstract ExportRequestType getType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java new file mode 100644 index 0000000000..07e5c946ba --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +public enum ExportRequestType { + SINGLE_ENTITY, + ENTITY_LIST, + ENTITY_TYPE, + + ENTITY_FILTER, + ENTITY_QUERY +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java new file mode 100644 index 0000000000..fe07ba7534 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class SingleEntityExportRequest extends ExportRequest { + + private EntityId entityId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 772e010b2b..5d693e25b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java new file mode 100644 index 0000000000..cc2eeba618 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.importing.data.request; + +import lombok.Data; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; + +import java.util.List; + +@Data +public class ImportRequest { + + private List>> exportDataList; + private EntityImportSettings importSettings; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 1c031f50e4..9a79c26515 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; @@ -47,9 +48,11 @@ public class AssetImportService extends BaseEntityImportService { - entityActionService.onAssetCreatedOrUpdated(user, savedAsset, oldAsset == null); - }; + return super.getCallback(user, savedAsset, oldAsset).andThen(() -> { + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index bf3c126c92..9715fe008e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -20,8 +20,11 @@ import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasCustomerId; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -56,6 +59,8 @@ public abstract class BaseEntityImportService {}; + return () -> { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); + }; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 336d9f68f2..a03ee60569 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -48,9 +49,11 @@ public class CustomerImportService extends BaseEntityImportService { - entityActionService.onCustomerCreatedOrUpdated(user, savedCustomer); - }; + return super.getCallback(user, savedCustomer, oldCustomer).andThen(() -> { + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index af24576d34..e80e699c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -23,7 +23,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; -import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -35,14 +35,12 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -114,9 +112,11 @@ public class DashboardImportService extends BaseEntityImportService { - entityActionService.onDashboardCreatedOrUpdated(user, savedDashboard, oldDashboard == null); - }; + return super.getCallback(user, savedDashboard, oldDashboard).andThen(() -> { + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 4212083dea..7484fb98ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -17,17 +17,14 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; @Service @@ -36,7 +33,6 @@ import org.thingsboard.server.utils.ThrowingRunnable; public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; - private final TbClusterService clusterService; @Override protected void setOwner(TenantId tenantId, Device device, NewIdProvider idProvider) { @@ -60,9 +56,9 @@ public class DeviceImportService extends BaseEntityImportService { + return super.getCallback(user, savedDevice, oldDevice).andThen(() -> { clusterService.onDeviceUpdated(savedDevice, oldDevice); - }; + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index f9d8d2ddd1..8db608fb42 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -19,20 +19,26 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; import org.thingsboard.server.utils.ThrowingRunnable; +import java.util.Objects; + @Service @TbCoreComponent @RequiredArgsConstructor public class DeviceProfileImportService extends BaseEntityImportService { private final DeviceProfileService deviceProfileService; + private final OtaPackageStateService otaPackageStateService; @Override protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, NewIdProvider idProvider) { @@ -50,9 +56,16 @@ public class DeviceProfileImportService extends BaseEntityImportService { - - }; + return super.getCallback(user, savedDeviceProfile, oldDeviceProfile).andThen(() -> { + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index f6a96f24bf..ec8a0aa5a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -21,10 +21,13 @@ import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -83,9 +86,14 @@ public class RuleChainImportService extends BaseEntityImportService { - - }; + return super.getCallback(user, savedRuleChain, oldRuleChain).andThen(() -> { + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java index 6965a79d19..858ccdf422 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java @@ -21,4 +21,11 @@ public interface ThrowingRunnable { void run() throws ThingsboardException; + default ThrowingRunnable andThen(ThrowingRunnable after) { + return () -> { + this.run(); + after.run(); + }; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index f114af6625..9c3c8d3ff3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JavaType; import lombok.SneakyThrows; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; @@ -54,6 +55,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +@Ignore @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { From 1b48a98fa091b1e85c18aa53905d6f8e47af1f1e Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 8 Apr 2022 13:53:28 +0300 Subject: [PATCH 039/178] Save relations after all other entities are saved on import; refactoring --- .../EntitiesExportImportController.java | 15 +-- .../DefaultEntitiesExportImportService.java | 56 +++++++--- .../sync/EntitiesExportImportService.java | 8 +- .../sync/importing/EntityImportService.java | 2 + .../{ => data}/EntityImportResult.java | 25 +++-- .../{ => data}/EntityImportSettings.java | 2 +- .../importing/data/request/ImportRequest.java | 2 +- .../importing/impl/AssetImportService.java | 12 +-- .../impl/BaseEntityImportService.java | 102 +++++++++--------- .../importing/impl/CustomerImportService.java | 14 +-- .../impl/DashboardImportService.java | 12 +-- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 22 ++-- .../impl/RuleChainImportService.java | 18 ++-- ...EntitiesExportImportControllerSqlTest.java | 2 +- .../java/org/thingsboard/server/dao/Dao.java | 2 + .../server/dao/asset/BaseAssetService.java | 2 +- .../server/dao/sql/JpaAbstractDao.java | 8 ++ 18 files changed, 169 insertions(+), 143 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/importing/{ => data}/EntityImportResult.java (60%) rename application/src/main/java/org/thingsboard/server/service/sync/importing/{ => data}/EntityImportSettings.java (94%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 366eaec41a..e1f5dd4525 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -46,16 +46,13 @@ import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExp import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.UUID; -import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @@ -163,15 +160,7 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> importResults = exportImportService.importEntities(user, - importRequest.getExportDataList(), importRequest.getImportSettings()); - - for (EntityImportResult> entityImportResult : importResults) { - if (entityImportResult.getCallback() != null) { - entityImportResult.getCallback().run(); - } - } - return importResults; + return exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 9bca5f8bc9..5df3cdde0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -35,12 +36,13 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; @@ -48,10 +50,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor +@Slf4j public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); @@ -77,7 +82,37 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + List>> importResults = new ArrayList<>(); + + for (EntityExportData> exportData : exportDataList) { + EntityImportResult> importResult = importEntity(user, exportData, importSettings); + importResults.add(importResult); + } + + for (ThrowingRunnable saveReferencesCallback : importResults.stream() + .map(EntityImportResult::getSaveReferencesCallback) + .filter(Objects::nonNull) + .collect(Collectors.toList())) { + saveReferencesCallback.run(); + } + + importResults.stream() + .map(EntityImportResult::getPushEventsCallback) + .filter(Objects::nonNull) + .forEach(pushEventsCallback -> { + try { + pushEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + + return importResults; + } + + private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } @@ -85,22 +120,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - // TODO [viacheslav]: might throw DataIntegrityViolationException with cause of ConstraintViolationException: need to give normal error return importService.importEntity(user, exportData, importSettings); } - @Transactional(rollbackFor = Exception.class) - @Override - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); - - List>> importResults = new ArrayList<>(); - for (EntityExportData> exportData : exportDataList) { - importResults.add(importEntity(user, exportData, importSettings)); - } - return importResults; - } - @Override public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 3fa9b13406..b0690ea9ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -19,10 +19,10 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.List; @@ -30,8 +30,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException; - List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index e3e40eea2a..6c82530273 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index 90d47c4378..e51048c433 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -13,19 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.utils.ThrowingRunnable; -@Data public class EntityImportResult> { + @Getter @Setter private E savedEntity; + @Getter @Setter private E oldEntity; - @JsonIgnore - private transient ThrowingRunnable callback; // to call when entity is successfully saved and transaction is committed + + @Getter @JsonIgnore + private ThrowingRunnable saveReferencesCallback = () -> {}; + @Getter @JsonIgnore + private ThrowingRunnable pushEventsCallback = () -> {}; + + public void addSaveReferencesCallback(ThrowingRunnable callback) { + this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); + } + + public void addPushEventsCallback(ThrowingRunnable callback) { + this.pushEventsCallback = this.pushEventsCallback.andThen(callback); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 9145762bef..1b5a93c0d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.importing.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java index cc2eeba618..6e857958d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -19,7 +19,7 @@ import lombok.Data; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 9a79c26515..0c8587270a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.AssetExportData; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -47,12 +46,11 @@ public class AssetImportService extends BaseEntityImportService { - if (oldAsset != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) { + super.onEntitySaved(user, savedAsset, oldAsset); + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 9715fe008e..b7602a53bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -37,10 +37,9 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.LinkedList; @@ -81,12 +80,13 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); - importResult.setCallback(callback); + + processAfterSaved(user, importResult, exportData, idProvider, importSettings); + return importResult; } @@ -94,60 +94,66 @@ public abstract class BaseEntityImportService newRelations = new LinkedList<>(); + protected void processAfterSaved(SecurityUser user, EntityImportResult importResult, D exportData, + NewIdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { + E savedEntity = importResult.getSavedEntity(); + E oldEntity = importResult.getOldEntity(); - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> relation.setTo(savedEntity.getId())) - .collect(Collectors.toList())); + importResult.addSaveReferencesCallback(() -> { + List newRelations = new LinkedList<>(); - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); - relationService.deleteRelation(user.getTenantId(), existingRelation); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + newRelations.addAll(exportData.getInboundRelations().stream() + .peek(relation -> relation.setTo(savedEntity.getId())) + .collect(Collectors.toList())); + + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { + for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - } - if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { - newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> relation.setFrom(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); - relationService.deleteRelation(user.getTenantId(), existingRelation); + if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { + newRelations.addAll(exportData.getOutboundRelations().stream() + .peek(relation -> relation.setFrom(savedEntity.getId())) + .collect(Collectors.toList())); + + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { + for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - } - for (EntityRelation relation : newRelations) { - HasId otherEntity = null; - if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(otherEntity.getId()); - } - if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(otherEntity.getId()); - } - if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); - } + for (EntityRelation relation : newRelations) { + HasId otherEntity = null; + if (!relation.getTo().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); + relation.setTo(otherEntity.getId()); + } + if (!relation.getFrom().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); + relation.setFrom(otherEntity.getId()); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); + } - relationService.saveRelation(user.getTenantId(), relation); - } + relationService.saveRelation(user.getTenantId(), relation); + } + }); - return getCallback(user, savedEntity, oldEntity); + importResult.addPushEventsCallback(() -> { + onEntitySaved(user, savedEntity, oldEntity); + }); } - protected ThrowingRunnable getCallback(SecurityUser user, E savedEntity, E oldEntity) { - return () -> { - entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), - oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); - }; + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, + savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index a03ee60569..4d2cfdfffb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -20,15 +20,12 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -48,12 +45,11 @@ public class CustomerImportService extends BaseEntityImportService { - if (oldCustomer != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) { + super.onEntitySaved(user, savedCustomer, oldCustomer); + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index e80e699c44..6463fe581c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -35,7 +35,6 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; @@ -111,12 +110,11 @@ public class DashboardImportService extends BaseEntityImportService { - if (oldDashboard != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) { + super.onEntitySaved(user, savedDashboard, oldDashboard); + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 7484fb98ba..850fea2baa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -25,7 +25,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -55,10 +54,9 @@ public class DeviceImportService extends BaseEntityImportService { - clusterService.onDeviceUpdated(savedDevice, oldDevice); - }); + protected void onEntitySaved(SecurityUser user, Device savedDevice, Device oldDevice) { + super.onEntitySaved(user, savedDevice, oldDevice); + clusterService.onDeviceUpdated(savedDevice, oldDevice); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index 8db608fb42..b68ae2a776 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Objects; @@ -55,17 +54,16 @@ public class DeviceProfileImportService extends BaseEntityImportService { - clusterService.onDeviceProfileChange(savedDeviceProfile, null); - clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), - oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), - oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); - otaPackageStateService.update(savedDeviceProfile, - oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), - oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); - }); + protected void onEntitySaved(SecurityUser user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) { + super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile); + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index ec8a0aa5a2..16f11171ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -32,7 +32,6 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.Optional; @@ -85,15 +84,14 @@ public class RuleChainImportService extends BaseEntityImportService { - if (savedRuleChain.getType() == RuleChainType.CORE) { - clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), - oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, RuleChain savedRuleChain, RuleChain oldRuleChain) { + super.onEntitySaved(user, savedRuleChain, oldRuleChain); + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 9c3c8d3ff3..603a43bf8c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -48,7 +48,7 @@ import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import java.util.List; diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 038be61d21..663c19348d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -37,6 +37,8 @@ public interface Dao { T save(TenantId tenantId, T t); + T saveAndFlush(TenantId tenantId, T t); + boolean removeById(TenantId tenantId, UUID id); void removeAllByIds(Collection ids); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index bf8d75d27a..2c379ae91e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -117,7 +117,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ assetValidator.validate(asset, Asset::getTenantId); Asset savedAsset; try { - savedAsset = assetDao.save(asset.getTenantId(), asset); + savedAsset = assetDao.saveAndFlush(asset.getTenantId(), asset); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("asset_name_unq_key")) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 3c996ff6f5..ad6bb5ccec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -67,6 +67,14 @@ public abstract class JpaAbstractDao, D> return DaoUtil.getData(entity); } + @Override + @Transactional + public D saveAndFlush(TenantId tenantId, D domain) { + D d = save(tenantId, domain); + getRepository().flush(); + return d; + } + @Override public D findById(TenantId tenantId, UUID key) { log.debug("Get entity by key {}", key); From a35b16d7cb75d2fc73bc4696634fd1e931739587 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 8 Apr 2022 18:24:50 +0300 Subject: [PATCH 040/178] Get rid of boilerplate for export --- .../EntitiesExportImportController.java | 18 +++- .../DefaultEntitiesExportImportService.java | 40 +++++---- .../sync/exporting/EntityExportService.java | 3 - .../sync/exporting/data/AssetExportData.java | 32 ------- .../exporting/data/CustomerExportData.java | 32 ------- .../exporting/data/DashboardExportData.java | 32 ------- .../sync/exporting/data/DeviceExportData.java | 6 -- .../data/DeviceProfileExportData.java | 32 ------- .../sync/exporting/data/EntityExportData.java | 31 ++++--- .../exporting/data/RuleChainExportData.java | 6 -- .../data/request/EntityExportSettings.java | 6 ++ .../exporting/impl/AssetExportService.java | 39 -------- .../impl/BaseEntityExportService.java | 60 ++----------- .../exporting/impl/CustomerExportService.java | 39 -------- .../impl/DashboardExportService.java | 39 -------- .../impl/DefaultEntityExportService.java | 90 +++++++++++++++++++ .../exporting/impl/DeviceExportService.java | 6 +- .../impl/DeviceProfileExportService.java | 39 -------- .../impl/RuleChainExportService.java | 6 +- .../importing/data/EntityImportResult.java | 3 + .../importing/impl/AssetImportService.java | 9 +- .../impl/BaseEntityImportService.java | 3 +- .../importing/impl/CustomerImportService.java | 9 +- .../impl/DashboardImportService.java | 9 +- .../importing/impl/DeviceImportService.java | 3 +- .../impl/DeviceProfileImportService.java | 9 +- .../impl/RuleChainImportService.java | 3 +- ...aseEntitiesExportImportControllerTest.java | 62 ++++++++++++- ...EntitiesExportImportControllerSqlTest.java | 60 ++++--------- 29 files changed, 277 insertions(+), 449 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index e1f5dd4525..9f9e90b0ca 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -52,6 +53,7 @@ import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -61,6 +63,7 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @RequestMapping("/api/entities") @TbCoreComponent @RequiredArgsConstructor +@Slf4j public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; @@ -160,7 +163,20 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - return exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + List>> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + + importResults.stream() + .map(EntityImportResult::getPushEventsCallback) + .filter(Objects::nonNull) + .forEach(pushEventsCallback -> { + try { + pushEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + + return importResults; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 5df3cdde0c..eefa396d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -39,6 +38,8 @@ import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; @@ -56,7 +57,6 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -@Slf4j public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); @@ -98,17 +98,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS saveReferencesCallback.run(); } - importResults.stream() - .map(EntityImportResult::getPushEventsCallback) - .filter(Objects::nonNull) - .forEach(pushEventsCallback -> { - try { - pushEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send event for entity", e); - } - }); - return importResults; } @@ -188,16 +177,31 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return daos.get(entityType); } + @Autowired - private void setServices(Collection> exportServices, - Collection> importServices, - Collection> daos) { - exportServices.forEach(entityExportService -> { - this.exportServices.put(entityExportService.getEntityType(), entityExportService); + private void setExportServices(DefaultEntityExportService defaultExportService, + Collection> exportServices) { + exportServices.stream() + .sorted(Comparator.comparing(exportService -> exportService.getSupportedEntityTypes().size(), Comparator.reverseOrder())) + .forEach(exportService -> { + exportService.getSupportedEntityTypes().forEach(entityType -> { + this.exportServices.put(entityType, exportService); + }); + }); + SUPPORTED_ENTITY_TYPES.forEach(entityType -> { + this.exportServices.putIfAbsent(entityType, defaultExportService); }); + } + + @Autowired + private void setImportServices(Collection> importServices) { importServices.forEach(entityImportService -> { this.importServices.put(entityImportService.getEntityType(), entityImportService); }); + } + + @Autowired + private void setDaos(Collection> daos) { daos.forEach(dao -> { if (dao.getEntityType() != null) { this.daos.put(dao.getEntityType(), dao); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index c09cbcd36c..73f946e722 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.exporting; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -27,6 +26,4 @@ public interface EntityExportService { - - @Override - public EntityType getEntityType() { - return EntityType.ASSET; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java deleted file mode 100644 index 08888f4a35..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class CustomerExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.CUSTOMER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java deleted file mode 100644 index 6f101816c9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class DashboardExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.DASHBOARD; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 977bd22f82..30d84c0a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) @@ -27,9 +26,4 @@ public class DeviceExportData extends EntityExportData { private DeviceCredentials credentials; - @Override - public EntityType getEntityType() { - return EntityType.DEVICE; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java deleted file mode 100644 index 7b71749be4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class DeviceProfileExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.DEVICE_PROFILE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index 51b5615c9e..bf44b0dc37 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -15,39 +15,48 @@ */ package org.thingsboard.server.service.sync.exporting.data; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.rule.RuleChain; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class, visible = true) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), - @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), - @Type(name = "ASSET", value = AssetExportData.class), - @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), - @Type(name = "DASHBOARD", value = DashboardExportData.class), - @Type(name = "CUSTOMER", value = CustomerExportData.class) + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) }) @JsonInclude(JsonInclude.Include.NON_NULL) @Data -public abstract class EntityExportData> { +public class EntityExportData> { + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) + @JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class) + }) private E entity; + private EntityType entityType; + private List inboundRelations; private List outboundRelations; - @JsonIgnore - public abstract EntityType getEntityType(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index 9f59aa4401..025459f27e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -27,9 +26,4 @@ public class RuleChainExportData extends EntityExportData { private RuleChainMetaData metaData; - @Override - public EntityType getEntityType() { - return EntityType.RULE_CHAIN; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index 01e4f8d068..ddf2e43bcf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -15,9 +15,15 @@ */ package org.thingsboard.server.service.sync.exporting.data.request; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor +@Builder public class EntityExportSettings { private boolean exportInboundRelations; private boolean exportOutboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java deleted file mode 100644 index b4f506a1f5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.AssetExportData; - -@Service -@TbCoreComponent -public class AssetExportService extends BaseEntityExportService { - - @Override - protected AssetExportData newExportData() { - return new AssetExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.ASSET; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 5d693e25b6..7b5f758f93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -15,71 +15,29 @@ */ package org.thingsboard.server.service.sync.exporting.impl; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import java.util.List; - -public abstract class BaseEntityExportService, D extends EntityExportData> implements EntityExportService { +import java.util.Set; - @Autowired @Lazy - private ExportableEntitiesService exportableEntitiesService; - @Autowired - private RelationService relationService; +public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - public final D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - D exportData = newExportData(); - - E entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); - if (entity == null) { - throw new IllegalArgumentException("Entity not found"); - } - exportableEntitiesService.checkPermission(user, entity, getEntityType(), Operation.READ); - - exportData.setEntity(entity); - setRelatedEntities(user.getTenantId(), entity, exportData); - setAdditionalExportData(user, entity, exportData, exportSettings); - - return exportData; + protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + setRelatedEntities(user.getTenantId(), entity, (D) exportData); + super.setAdditionalExportData(user, entity, exportData, exportSettings); } protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} - protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { - if (exportSettings.isExportInboundRelations()) { - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (inboundRelations != null) { - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - } - exportData.setInboundRelations(inboundRelations); - } - if (exportSettings.isExportOutboundRelations()) { - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (outboundRelations != null) { - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - } - exportData.setOutboundRelations(outboundRelations); - } - } - protected abstract D newExportData(); + public abstract Set getSupportedEntityTypes(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java deleted file mode 100644 index c90d21e573..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; - -@Service -@TbCoreComponent -public class CustomerExportService extends BaseEntityExportService { - - @Override - protected CustomerExportData newExportData() { - return new CustomerExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.CUSTOMER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java deleted file mode 100644 index 32584ca917..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; - -@Service -@TbCoreComponent -public class DashboardExportService extends BaseEntityExportService { - - @Override - protected DashboardExportData newExportData() { - return new DashboardExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.DASHBOARD; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java new file mode 100644 index 0000000000..4fe09e9322 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; + +import java.util.List; + +@Service +@TbCoreComponent +@Primary +public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + + @Override + public final D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + D exportData = newExportData(); + + E entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (entity == null) { + throw new IllegalArgumentException(entityId.getEntityType() + " [" + entityId.getId() + "] not found"); + } + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + + exportData.setEntity(entity); + exportData.setEntityType(entityId.getEntityType()); + setAdditionalExportData(user, entity, exportData, exportSettings); + + return exportData; + } + + protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + if (exportSettings.isExportInboundRelations()) { + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + if (inboundRelations != null) { + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + } + exportData.setInboundRelations(inboundRelations); + } + if (exportSettings.isExportOutboundRelations()) { + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + if (outboundRelations != null) { + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); + } + } + exportData.setOutboundRelations(outboundRelations); + } + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index d31a859013..da1e4f3a96 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import java.util.Set; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -43,8 +45,8 @@ public class DeviceExportService extends BaseEntityExportService getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java deleted file mode 100644 index b3fd65363b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; - -@Service -@TbCoreComponent -public class DeviceProfileExportService extends BaseEntityExportService { - - @Override - protected DeviceProfileExportData newExportData() { - return new DeviceProfileExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.DEVICE_PROFILE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java index 846d4006d0..e814cd6ce7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import java.util.Set; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -43,8 +45,8 @@ public class RuleChainExportService extends BaseEntityExportService getSupportedEntityTypes() { + return Set.of(EntityType.RULE_CHAIN); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index e51048c433..7160a483b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.utils.ThrowingRunnable; @@ -27,6 +28,8 @@ public class EntityImportResult> private E savedEntity; @Getter @Setter private E oldEntity; + @Getter @Setter + private EntityType entityType; @Getter @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 0c8587270a..02eaae27b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -20,17 +20,18 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; @Service @TbCoreComponent @RequiredArgsConstructor -public class AssetImportService extends BaseEntityImportService { +public class AssetImportService extends BaseEntityImportService> { private final AssetService assetService; @@ -41,12 +42,12 @@ public class AssetImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { return assetService.saveAsset(asset); } @Override - protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) { + protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) throws ThingsboardException { super.onEntitySaved(user, savedAsset, oldAsset); if (oldAsset != null) { entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index b7602a53bf..b576ae1e60 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -84,6 +84,7 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); + importResult.setEntityType(getEntityType()); processAfterSaved(user, importResult, exportData, idProvider, importSettings); @@ -150,7 +151,7 @@ public abstract class BaseEntityImportService { +public class CustomerImportService extends BaseEntityImportService> { private final CustomerService customerService; @@ -40,12 +41,12 @@ public class CustomerImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { return customerService.saveCustomer(customer); } @Override - protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) { + protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException { super.onEntitySaved(user, savedCustomer, oldCustomer); if (oldCustomer != null) { entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 6463fe581c..3e06480a2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +35,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.Collections; import java.util.HashSet; @@ -46,7 +47,7 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -public class DashboardImportService extends BaseEntityImportService { +public class DashboardImportService extends BaseEntityImportService> { private final DashboardService dashboardService; @@ -58,7 +59,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) .filter(JsonNode::isObject) @@ -110,7 +111,7 @@ public class DashboardImportService extends BaseEntityImportService { +public class DeviceProfileImportService extends BaseEntityImportService> { private final DeviceProfileService deviceProfileService; private final OtaPackageStateService otaPackageStateService; @@ -45,7 +46,7 @@ public class DeviceProfileImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.get(DeviceProfile::getDefaultRuleChainId)); deviceProfile.setDefaultDashboardId(idProvider.get(DeviceProfile::getDefaultDashboardId)); deviceProfile.setFirmwareId(idProvider.get(DeviceProfile::getFirmwareId)); @@ -54,7 +55,7 @@ public class DeviceProfileImportService extends BaseEntityImportService, I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(entityId); + exportRequest.setExportSettings(new EntityExportSettings()); + return (EntityExportData) exportEntities(exportRequest).get(0); + } + + protected List>> exportEntities(ExportRequest exportRequest) throws Exception { + return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>>() {}); + } + + protected List>> exportEntities(List exportRequests) throws Exception { + return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>>() {}); + } + + + protected , I extends EntityId> EntityImportResult importEntity(EntityExportData exportData) throws Exception { + return (EntityImportResult) importEntities(List.of((EntityExportData>)exportData)).get(0); + } + + protected List>> importEntities(List>> exportDataList) throws Exception { + ImportRequest importRequest = new ImportRequest(); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(true) + .build()); + importRequest.setExportDataList(exportDataList); + return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>>() { + @Override + public Type getType() { + return mapper.getTypeFactory().constructCollectionType(List.class, + mapper.getTypeFactory().constructParametricType(EntityImportResult.class, + exportDataList.get(0).getEntity().getClass())); + } + }); + } + + protected T getResponse(ResultActions resultActions, TypeReference typeReference) throws Exception { + try { + return readResponse(resultActions.andExpect(status().isOk()), typeReference); + } catch (AssertionError e) { + throw new AssertionError(readResponse(resultActions, String.class), e); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 603a43bf8c..ccbf311bef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -15,15 +15,10 @@ */ package org.thingsboard.server.controller.sql; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JavaType; -import lombok.SneakyThrows; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -33,7 +28,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -50,12 +45,8 @@ import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@Ignore @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { @@ -110,7 +101,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(asset); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, asset, tenantId2, importResult); checkImportedAssetData(asset, importResult.getSavedEntity()); @@ -122,7 +113,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); EntityExportData exportData = exportSingleEntity(asset.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, asset, tenantId1, importResult); checkImportedAssetData(asset, importResult.getSavedEntity()); @@ -138,8 +129,8 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData assetExportData = exportSingleEntity(asset.getId()); logInAsTenantAdmin2(); - Customer importedCustomer = importEntities(List.of(customerExportData)).get(0).getSavedEntity(); - Asset importedAsset = importEntities(List.of(assetExportData)).get(0).getSavedEntity(); + Customer importedCustomer = importEntity(customerExportData).getSavedEntity(); + Asset importedAsset = importEntity(assetExportData).getSavedEntity(); assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); } @@ -150,7 +141,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Customer customer = createCustomer(tenantId1, "My customer"); Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); - Asset importedAsset = this.importEntities(List.of(exportSingleEntity(asset.getId()))).get(0).getSavedEntity(); + Asset importedAsset = importEntity(this.exportSingleEntity(asset.getId())).getSavedEntity(); assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); } @@ -171,7 +162,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(customer); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, customer, tenantId2, importResult); checkImportedCustomerData(customer, importResult.getSavedEntity()); @@ -183,7 +174,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); EntityExportData exportData = exportSingleEntity(customer.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, customer, tenantId1, importResult); checkImportedCustomerData(customer, importResult.getSavedEntity()); @@ -214,12 +205,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); logInAsTenantAdmin2(); - EntityImportResult profileImportResult = importEntities(List.of(profileExportData)).get(0); + EntityImportResult profileImportResult = importEntity(profileExportData); checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); - EntityImportResult deviceImportResult = importEntities(List.of(deviceExportData)).get(0); + EntityImportResult deviceImportResult = importEntity(deviceExportData); Device importedDevice = deviceImportResult.getSavedEntity(); checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); checkImportedDeviceData(device, importedDevice); @@ -242,7 +233,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData deviceExportData = exportSingleEntity(device.getId()); - EntityImportResult importResult = importEntities(List.of(deviceExportData)).get(0); + EntityImportResult importResult = importEntity(deviceExportData); Device importedDevice = importResult.getSavedEntity(); checkImportedEntity(tenantId1, device, tenantId1, importResult); @@ -277,7 +268,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(((RuleChainExportData) exportData).getMetaData()).isEqualTo(metaData); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); @@ -293,7 +284,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData exportData = exportSingleEntity(ruleChain.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); @@ -330,7 +321,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(dashboard); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } @@ -342,7 +333,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData exportData = exportSingleEntity(dashboard.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } @@ -376,27 +367,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } - @SneakyThrows - private > List> importEntities(List> exportDataList) { - String requestJson = mapper.writerFor(new TypeReference>>() {}).writeValueAsString(exportDataList); - ResultActions resultActions = doPost("/api/entities/import", (Object) requestJson); - - try { - String responseJson = resultActions.andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); - JavaType type = mapper.getTypeFactory().constructCollectionType(List.class, - mapper.getTypeFactory().constructParametricType(EntityImportResult.class, exportDataList.get(0).getEntity().getClass())); - return mapper.readValue(responseJson, type); - } catch (AssertionError e) { - throw new AssertionError(readResponse(resultActions, String.class), e); - } - } - - private , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { - return readResponse(doPost("/api/entities/export/" + entityId.getEntityType() + "/" + entityId.getId().toString()) - .andExpect(status().isOk()), new TypeReference>() {}); - } - - private void logInAsTenantAdmin1() throws Exception { login(tenantAdmin1.getEmail(), "12345678"); } From c04a0ff3a6fbef52c93a40bd56a9eb26fb3aea63 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 12 Apr 2022 13:04:25 +0300 Subject: [PATCH 041/178] More tests for export/import api; improvements --- .../EntitiesExportImportController.java | 45 +- .../DefaultEntitiesExportImportService.java | 10 +- .../sync/EntitiesExportImportService.java | 2 +- .../sync/exporting/data/EntityExportData.java | 19 +- ...a => CustomEntityFilterExportRequest.java} | 4 +- ...va => CustomEntityQueryExportRequest.java} | 4 +- .../exporting/data/request/ExportRequest.java | 6 +- .../data/request/ExportRequestType.java | 4 +- .../importing/data/EntityImportResult.java | 21 +- .../importing/data/EntityImportSettings.java | 2 + .../importing/data/request/ImportRequest.java | 2 +- .../impl/BaseEntityImportService.java | 33 +- .../impl/DashboardImportService.java | 4 +- .../impl/RuleChainImportService.java | 4 +- .../server/utils/JsonTbEntity.java | 47 ++ ...aseEntitiesExportImportControllerTest.java | 197 ++++- ...EntitiesExportImportControllerSqlTest.java | 731 ++++++++++++++---- .../server/common/data/ExportableEntity.java | 3 + 18 files changed, 875 insertions(+), 263 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/{EntityFilterExportRequest.java => CustomEntityFilterExportRequest.java} (89%) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/{EntityQueryExportRequest.java => CustomEntityQueryExportRequest.java} (89%) create mode 100644 application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 9f9e90b0ca..4a1badd38e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,13 +17,13 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DataAccessException; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -41,9 +41,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; @@ -71,7 +71,7 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { + public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { return exportEntitiesByRequest(user, exportRequest); @@ -82,10 +82,10 @@ public class EntitiesExportImportController extends BaseController { @PostMapping(value = "/export", params = {"multiple"}) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { + public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> exportDataList = new ArrayList<>(); + List> exportDataList = new ArrayList<>(); for (ExportRequest exportRequest : exportRequests) { exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); } @@ -96,10 +96,10 @@ public class EntitiesExportImportController extends BaseController { } - private List>> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { + private List> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { List entitiesIds = findEntitiesForRequest(user, request); - List>> exportDataList = new ArrayList<>(); + List> exportDataList = new ArrayList<>(); for (EntityId entityId : entitiesIds) { exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); } @@ -122,15 +122,15 @@ public class EntitiesExportImportController extends BaseController { CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); } - case ENTITY_FILTER: { - EntityFilterExportRequest exportRequest = (EntityFilterExportRequest) request; + case CUSTOM_ENTITY_FILTER: { + CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; EntityFilter filter = exportRequest.getFilter(); CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); } - case ENTITY_QUERY:{ - EntityQueryExportRequest exportRequest = (EntityQueryExportRequest) request; + case CUSTOM_ENTITY_QUERY: { + CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; EntityDataQuery query = exportRequest.getQuery(); CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); @@ -153,24 +153,29 @@ public class EntitiesExportImportController extends BaseController { } private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() - .map(EntityData::getEntityId) - .collect(Collectors.toList()); + try { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + } catch (DataAccessException e) { + log.error("Failed to find entity data by query: {}", e.getMessage()); + throw new IllegalArgumentException("Entity filter cannot be processed"); + } } @PostMapping("/import") - public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { + public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); importResults.stream() - .map(EntityImportResult::getPushEventsCallback) + .map(EntityImportResult::getSendEventsCallback) .filter(Objects::nonNull) - .forEach(pushEventsCallback -> { + .forEach(sendEventsCallback -> { try { - pushEventsCallback.run(); + sendEventsCallback.run(); } catch (Exception e) { log.error("Failed to send event for entity", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index eefa396d06..cfb895c36a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -67,7 +67,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, - EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD + EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE ); @@ -82,12 +82,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); - List>> importResults = new ArrayList<>(); + List> importResults = new ArrayList<>(); - for (EntityExportData> exportData : exportDataList) { - EntityImportResult> importResult = importEntity(user, exportData, importSettings); + for (EntityExportData exportData : exportDataList) { + EntityImportResult importResult = importEntity(user, exportData, importSettings); importResults.add(importResult); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index b0690ea9ca..eaaf01ba2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -30,6 +30,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; + List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index bf44b0dc37..4ab6dd57b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -21,21 +21,16 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.utils.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class, visible = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) @@ -44,15 +39,7 @@ import java.util.List; @Data public class EntityExportData> { - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) - @JsonSubTypes({ - @Type(name = "DEVICE", value = Device.class), - @Type(name = "RULE_CHAIN", value = RuleChain.class), - @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), - @Type(name = "ASSET", value = Asset.class), - @Type(name = "DASHBOARD", value = Dashboard.class), - @Type(name = "CUSTOMER", value = Customer.class) - }) + @JsonTbEntity private E entity; private EntityType entityType; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java index 00ef0604e5..02cbca98d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.query.EntityFilter; @EqualsAndHashCode(callSuper = true) @Data -public class EntityFilterExportRequest extends ExportRequest { +public class CustomEntityFilterExportRequest extends ExportRequest { private EntityFilter filter; private int page; @@ -31,7 +31,7 @@ public class EntityFilterExportRequest extends ExportRequest { @Override public ExportRequestType getType() { - return ExportRequestType.ENTITY_FILTER; + return ExportRequestType.CUSTOM_ENTITY_FILTER; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java index 85232821d7..76d258e891 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java @@ -22,14 +22,14 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; @EqualsAndHashCode(callSuper = true) @Data -public class EntityQueryExportRequest extends ExportRequest { +public class CustomEntityQueryExportRequest extends ExportRequest { private EntityDataQuery query; private CustomerId customerId; @Override public ExportRequestType getType() { - return ExportRequestType.ENTITY_QUERY; + return ExportRequestType.CUSTOM_ENTITY_QUERY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java index c140786d06..af0f161227 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.exporting.data.request; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -27,15 +26,14 @@ import lombok.Data; @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), - @Type(name = "ENTITY_FILTER", value = EntityFilterExportRequest.class), - @Type(name = "ENTITY_QUERY", value = EntityQueryExportRequest.class) + @Type(name = "CUSTOM_ENTITY_FILTER", value = CustomEntityFilterExportRequest.class), + @Type(name = "CUSTOM_ENTITY_QUERY", value = CustomEntityQueryExportRequest.class) }) @Data public abstract class ExportRequest { private EntityExportSettings exportSettings; - @JsonIgnore public abstract ExportRequestType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java index 07e5c946ba..c36ac567e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java @@ -20,6 +20,6 @@ public enum ExportRequestType { ENTITY_LIST, ENTITY_TYPE, - ENTITY_FILTER, - ENTITY_QUERY + CUSTOM_ENTITY_FILTER, + CUSTOM_ENTITY_QUERY } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index 7160a483b3..62c574d5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -16,32 +16,33 @@ package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.utils.JsonTbEntity; import org.thingsboard.server.utils.ThrowingRunnable; +@Data public class EntityImportResult> { - @Getter @Setter + + @JsonTbEntity private E savedEntity; - @Getter @Setter + @JsonTbEntity private E oldEntity; - @Getter @Setter private EntityType entityType; - @Getter @JsonIgnore + @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @Getter @JsonIgnore - private ThrowingRunnable pushEventsCallback = () -> {}; + @JsonIgnore + private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); } - public void addPushEventsCallback(ThrowingRunnable callback) { - this.pushEventsCallback = this.pushEventsCallback.andThen(callback); + public void addSendEventsCallback(ThrowingRunnable callback) { + this.sendEventsCallback = this.sendEventsCallback.andThen(callback); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 1b5a93c0d8..3b457e1f5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -26,8 +26,10 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean findExistingByName; + private boolean importInboundRelations; private boolean importOutboundRelations; private boolean removeExistingRelations; + private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java index 6e857958d6..7943e1a404 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -26,7 +26,7 @@ import java.util.List; @Data public class ImportRequest { - private List>> exportDataList; + private List> exportDataList; private EntityImportSettings importSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index b576ae1e60..97547868d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.audit.ActionType; @@ -76,6 +75,7 @@ public abstract class BaseEntityImportService { + importResult.addSendEventsCallback(() -> { onEntitySaved(user, savedEntity, oldEntity); }); } @@ -172,8 +172,6 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { - if (externalId == null || externalId.isNullUid()) return null; - return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); @@ -187,19 +185,18 @@ public abstract class BaseEntityImportService ALWAYS_UPDATE_REFERENCED_IDS = Set.of( - EntityType.RULE_CHAIN - ); - public ID get(Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() - || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return getInternalId(idExtractor.apply(this.entity)); } else { return idExtractor.apply(existingEntity); } } + public ID getInternal(ID externalId) { + return getInternalId(externalId); + } + public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() @@ -213,17 +210,15 @@ public abstract class BaseEntityImportService ID getInternalId(ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + HasId entity = findInternalEntity(user.getTenantId(), externalId); - if (entity != null) { - try { - exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); - } catch (ThingsboardException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - return entity.getId(); - } else { - return null; + try { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); } + return entity.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 3e06480a2f..fda6904421 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -74,7 +74,7 @@ public class DashboardImportService extends BaseEntityImportService { String uuid = matchResult.group(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.get(d -> entityId).toString(); + return idProvider.getInternal(entityId).toString(); }); ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); }); @@ -86,7 +86,7 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index f901fcfd17..5fef00cbed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -63,14 +63,14 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.get(rc -> new RuleChainId(otherRuleChainUuid)).toString() + idProvider.getInternal(new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternal(ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java new file mode 100644 index 0000000000..cfc38ff3c3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.rule.RuleChain; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@JacksonAnnotationsInside +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) +@JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class) +}) +public @interface JsonTbEntity { +} diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index 0d73ed9f09..a8a5bca534 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -17,6 +17,8 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.JacksonUtil; @@ -28,6 +30,10 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -35,19 +41,29 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; @@ -56,11 +72,12 @@ import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; -import java.lang.reflect.Type; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseEntitiesExportImportControllerTest extends AbstractControllerTest { @@ -68,6 +85,8 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon @Autowired protected DeviceService deviceService; @Autowired + protected OtaPackageService otaPackageService; + @Autowired protected DeviceProfileService deviceProfileService; @Autowired protected AssetService assetService; @@ -77,6 +96,45 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected RuleChainService ruleChainService; @Autowired protected DashboardService dashboardService; + @Autowired + protected RelationService relationService; + @Autowired + protected TenantService tenantService; + + protected TenantId tenantId1; + protected User tenantAdmin1; + + protected TenantId tenantId2; + protected User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { Device device = new Device(); @@ -91,13 +149,38 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return deviceService.saveDevice(device); } - protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { + protected OtaPackage createOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type) { + OtaPackage otaPackage = new OtaPackage(); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfileId); + otaPackage.setType(type); + otaPackage.setTitle("My " + type); + otaPackage.setVersion("v1.0"); + otaPackage.setFileName("filename.txt"); + otaPackage.setContentType("text/plain"); + otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + otaPackage.setDataSize(1L); + otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); + return otaPackageService.saveOtaPackage(otaPackage); + } + + protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, RuleChainId defaultRuleChainId, DashboardId defaultDashboardId, String name) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setTenantId(tenantId); deviceProfile.setName(name); deviceProfile.setDescription("dscrptn"); deviceProfile.setType(DeviceProfileType.DEFAULT); deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + deviceProfile.setDefaultRuleChainId(defaultRuleChainId); + deviceProfile.setDefaultDashboardId(defaultDashboardId); DeviceProfileData profileData = new DeviceProfileData(); profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); @@ -105,6 +188,14 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return deviceProfileService.saveDeviceProfile(deviceProfile); } + protected void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { Asset asset = new Asset(); asset.setTenantId(tenantId); @@ -116,6 +207,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return assetService.saveAsset(asset); } + protected void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + protected Customer createCustomer(TenantId tenantId, String name) { Customer customer = new Customer(); customer.setTenantId(tenantId); @@ -127,6 +225,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return customerService.saveCustomer(customer); } + protected void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { Dashboard dashboard = new Dashboard(); dashboard.setTenantId(tenantId); @@ -142,6 +247,16 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return dashboard; } + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + protected RuleChain createRuleChain(TenantId tenantId, String name) { RuleChain ruleChain = new RuleChain(); ruleChain.setTenantId(tenantId); @@ -178,6 +293,50 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); } + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + protected EntityRelation createRelation(EntityId from, EntityId to) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(from); + relation.setTo(to); + relation.setType(EntityRelation.MANAGES_TYPE); + relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + return relation; + } + + protected & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + } + } + protected , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); @@ -186,33 +345,30 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return (EntityExportData) exportEntities(exportRequest).get(0); } - protected List>> exportEntities(ExportRequest exportRequest) throws Exception { - return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>>() {}); + protected List> exportEntities(ExportRequest exportRequest) throws Exception { + return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>() {}); } - protected List>> exportEntities(List exportRequests) throws Exception { - return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>>() {}); + protected List> exportEntities(List exportRequests) throws Exception { + return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>() {}); } protected , I extends EntityId> EntityImportResult importEntity(EntityExportData exportData) throws Exception { - return (EntityImportResult) importEntities(List.of((EntityExportData>)exportData)).get(0); + return (EntityImportResult) importEntities(List.of((EntityExportData>) exportData)).get(0); } - protected List>> importEntities(List>> exportDataList) throws Exception { + protected List> importEntities(List> exportDataList) throws Exception { ImportRequest importRequest = new ImportRequest(); importRequest.setImportSettings(EntityImportSettings.builder() .updateReferencesToOtherEntities(true) .build()); importRequest.setExportDataList(exportDataList); - return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>>() { - @Override - public Type getType() { - return mapper.getTypeFactory().constructCollectionType(List.class, - mapper.getTypeFactory().constructParametricType(EntityImportResult.class, - exportDataList.get(0).getEntity().getClass())); - } - }); + return importEntities(importRequest); + } + + protected List> importEntities(ImportRequest importRequest) throws Exception { + return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>() {}); } protected T getResponse(ResultActions resultActions, TypeReference typeReference) throws Exception { @@ -223,4 +379,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon } } + + protected void logInAsTenantAdmin1() throws Exception { + login(tenantAdmin1.getEmail(), "12345678"); + } + + protected void logInAsTenantAdmin2() throws Exception { + login(tenantAdmin2.getEmail(), "12345678"); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index ccbf311bef..920d91e93d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -15,86 +15,81 @@ */ package org.thingsboard.server.controller.sql; -import org.junit.After; -import org.junit.Before; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.Streams; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.controller.BaseEntitiesExportImportControllerTest; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { - @Autowired - private TenantService tenantService; @Autowired private DeviceCredentialsService deviceCredentialsService; - @Autowired - private ExportableEntitiesService exportableEntitiesService; - - private TenantId tenantId1; - private User tenantAdmin1; - - private TenantId tenantId2; - private User tenantAdmin2; - - @Before - public void beforeEach() throws Exception { - loginSysAdmin(); - Tenant tenant1 = new Tenant(); - tenant1.setTitle("Tenant 1"); - tenant1.setEmail("tenant1@thingsboard.org"); - this.tenantId1 = tenantService.saveTenant(tenant1).getId(); - User tenantAdmin1 = new User(); - tenantAdmin1.setTenantId(tenantId1); - tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); - this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); - Tenant tenant2 = new Tenant(); - tenant2.setTitle("Tenant 2"); - tenant2.setEmail("tenant2@thingsboard.org"); - this.tenantId2 = tenantService.saveTenant(tenant2).getId(); - User tenantAdmin2 = new User(); - tenantAdmin2.setTenantId(tenantId2); - tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); - this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); - } - - @After - public void afterEach() { - tenantService.deleteTenant(tenantId1); - tenantService.deleteTenant(tenantId2); - } - + @SpyBean + private EntityActionService entityActionService; + @SpyBean + private TbClusterService clusterService; + @SpyBean + private OtaPackageStateService otaPackageStateService; @Test - public void testExportImportSingleAsset_betweenTenants() throws Exception { + public void testExportImportAsset_betweenTenants() throws Exception { logInAsTenantAdmin1(); Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); EntityExportData exportData = exportSingleEntity(asset.getId()); @@ -103,40 +98,24 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, asset, tenantId2, importResult); + checkImportedEntity(tenantId1, asset, tenantId2, importResult.getSavedEntity()); checkImportedAssetData(asset, importResult.getSavedEntity()); } @Test - public void testExportImportSingleAsset_sameTenant() throws Exception { + public void testExportImportAsset_sameTenant() throws Exception { logInAsTenantAdmin1(); Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); EntityExportData exportData = exportSingleEntity(asset.getId()); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, asset, tenantId1, importResult); + checkImportedEntity(tenantId1, asset, tenantId1, importResult.getSavedEntity()); checkImportedAssetData(asset, importResult.getSavedEntity()); } @Test - public void testExportImportAsset_withCustomer_betweenTenants() throws Exception { - logInAsTenantAdmin1(); - Customer customer = createCustomer(tenantId1, "My customer"); - Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); - - EntityExportData customerExportData = exportSingleEntity(customer.getId()); - EntityExportData assetExportData = exportSingleEntity(asset.getId()); - - logInAsTenantAdmin2(); - Customer importedCustomer = importEntity(customerExportData).getSavedEntity(); - Asset importedAsset = importEntity(assetExportData).getSavedEntity(); - - assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); - } - - @Test - public void testExportImportAsset_withCustomer_sameTenant() throws Exception { + public void testExportImportAsset_sameTenant_withCustomer() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantId1, "My customer"); Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); @@ -146,16 +125,9 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); } - private void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { - assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); - assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); - assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); - assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); - } - @Test - public void testExportImportSingleCustomer_betweenTenants() throws Exception { + public void testExportImportCustomer_betweenTenants() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); EntityExportData exportData = exportSingleEntity(customer.getId()); @@ -164,34 +136,27 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, customer, tenantId2, importResult); + checkImportedEntity(tenantId1, customer, tenantId2, importResult.getSavedEntity()); checkImportedCustomerData(customer, importResult.getSavedEntity()); } @Test - public void testExportImportSingleCustomer_sameTenant() throws Exception { + public void testExportImportCustomer_sameTenant() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); EntityExportData exportData = exportSingleEntity(customer.getId()); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, customer, tenantId1, importResult); + checkImportedEntity(tenantId1, customer, tenantId1, importResult.getSavedEntity()); checkImportedCustomerData(customer, importResult.getSavedEntity()); } - private void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { - assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); - assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); - assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); - assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); - } - @Test public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { logInAsTenantAdmin1(); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile of tenant 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); @@ -206,13 +171,13 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult profileImportResult = importEntity(profileExportData); - checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult.getSavedEntity()); checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); EntityImportResult deviceImportResult = importEntity(deviceExportData); Device importedDevice = deviceImportResult.getSavedEntity(); - checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult.getSavedEntity()); checkImportedDeviceData(device, importedDevice); assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); @@ -227,8 +192,14 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp @Test public void testExportImportDevice_sameTenant() throws Exception { logInAsTenantAdmin1(); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile v1.0"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile v1.0"); + OtaPackage firmware = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.FIRMWARE); + OtaPackage software = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.SOFTWARE); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + device.setFirmwareId(firmware.getId()); + device.setSoftwareId(software.getId()); + device = deviceService.saveDevice(device); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); EntityExportData deviceExportData = exportSingleEntity(device.getId()); @@ -236,29 +207,119 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityImportResult importResult = importEntity(deviceExportData); Device importedDevice = importResult.getSavedEntity(); - checkImportedEntity(tenantId1, device, tenantId1, importResult); + checkImportedEntity(tenantId1, device, tenantId1, importResult.getSavedEntity()); assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + assertThat(importedDevice.getFirmwareId()).isEqualTo(firmware.getId()); + assertThat(importedDevice.getSoftwareId()).isEqualTo(software.getId()); + } + + + @Test + public void testExportImportDashboard_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + assertThat(exportData.getEntity()).isEqualTo(dashboard); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntity(exportData); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { - assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); - assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); - assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); - assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); - assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + @Test + public void testExportImportDashboard_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + EntityImportResult importResult = importEntity(exportData); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private void checkImportedDeviceData(Device initialDevice, Device importedDevice) { - assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); - assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); - assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); - assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + @Test + public void testExportImportDashboard_betweenTenants_withCustomer_updated() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + logInAsTenantAdmin2(); + Dashboard importedDashboard = (Dashboard) importEntities(List.of(exportData)).get(0).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + EntityExportData customerExportData = exportSingleEntity(customer.getId()); + dashboardService.assignDashboardToCustomer(tenantId1, dashboard.getId(), customer.getId()); + exportData = exportSingleEntity(dashboard.getId()); + + logInAsTenantAdmin2(); + Customer importedCustomer = (Customer) importEntities(List.of(customerExportData)).get(0).getSavedEntity(); + importedDashboard = (Dashboard) importEntities(List.of(exportData)).get(0).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } + + @Test + public void testExportImportDashboard_betweenTenants_withEntityAliases() throws Exception { + logInAsTenantAdmin1(); + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + asset1.getId().toString() + "\",\n" + + "\t\t\t\t\"" + asset2.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + dashboard = dashboardService.saveDashboard(dashboard); + + EntityTypeExportRequest assetsExportRequest = new EntityTypeExportRequest(); + assetsExportRequest.setEntityType(EntityType.ASSET); + assetsExportRequest.setPageSize(10); + assetsExportRequest.setExportSettings(new EntityExportSettings()); + EntityTypeExportRequest dashboardsExportRequest = new EntityTypeExportRequest(); + dashboardsExportRequest.setEntityType(EntityType.DASHBOARD); + dashboardsExportRequest.setPageSize(10); + dashboardsExportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(List.of(assetsExportRequest, dashboardsExportRequest)); + + logInAsTenantAdmin2(); + Map>> importResults = importEntities(exportDataList).stream().collect(Collectors.groupingBy(EntityImportResult::getEntityType)); + Asset importedAsset1 = (Asset) importResults.get(EntityType.ASSET).get(0).getSavedEntity(); + Asset importedAsset2 = (Asset) importResults.get(EntityType.ASSET).get(1).getSavedEntity(); + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).get(0).getSavedEntity(); + + Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .map(JsonNode::asText).collect(Collectors.toSet()); + assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); + assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); } @Test - public void testExportImportSingleRuleChain_betweenTenants() throws Exception { + public void testExportImportRuleChain_betweenTenants() throws Exception { logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); @@ -272,12 +333,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); - checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult); + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult.getSavedEntity()); checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); } @Test - public void testExportImportSingleRuleChain_sameTenant() throws Exception { + public void testExportImportRuleChain_sameTenant() throws Exception { logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); @@ -288,91 +349,439 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); - checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult); + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult.getSavedEntity()); checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); } - private void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { - assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); - assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); - assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); - assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); - - assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); - assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); - for (int i = 0; i < initialMetaData.getNodes().size(); i++) { - RuleNode initialNode = initialMetaData.getNodes().get(i); - RuleNode importedNode = importedMetaData.getNodes().get(i); - assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); - assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); - assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); - assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); - assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); - } + + @Test + public void testExportImportBatch_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, customer.getId(), "A", "Customer 1 - Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Customer 1 - Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Customer 1 - Device 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), ruleChain.getId(), deviceProfile.getId(), dashboard.getId())); + List> exportDataList = exportEntities(exportRequest); + exportRequest.setEntitiesIds(List.of(device.getId())); + DeviceExportData deviceExportData = (DeviceExportData) exportEntities(exportRequest).get(0); + deviceExportData.getCredentials().setCredentialsId(RandomStringUtils.randomAlphanumeric(10)); + exportDataList.add(deviceExportData); + + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(true) + .build()); + importRequest.setExportDataList(exportDataList); + Map> importResults = importEntities(importRequest).stream() + .collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); + checkImportedEntity(tenantId1, customer, tenantId2, importedCustomer); + + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); + + RuleChain importedRuleChain = (RuleChain) importResults.get(EntityType.RULE_CHAIN).getSavedEntity(); + checkImportedEntity(tenantId1, ruleChain, tenantId2, importedRuleChain); + + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + assertThat(importedDashboard.getAssignedCustomers()).size().isOne(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importResults.get(EntityType.DEVICE_PROFILE).getSavedEntity(); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, importedDeviceProfile); + assertThat(importedDeviceProfile.getDefaultRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedDeviceProfile.getDefaultDashboardId()).isEqualTo(importedDashboard.getId()); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + assertThat(importedDevice.getCustomerId()).isEqualTo(importedCustomer.getId()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfile.getId()); } @Test - public void testExportImportSingleDashboard_betweenTenants() throws Exception { + public void testExportImportWithInboundRelations_betweenTenants() throws Exception { logInAsTenantAdmin1(); - Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); - EntityExportData exportData = exportSingleEntity(dashboard.getId()); - assertThat(exportData.getEntity()).isEqualTo(dashboard); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .exportOutboundRelations(false) + .build()); + List> exportDataList = exportEntities(exportRequest); + + EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); + assertThat(deviceExportData.getInboundRelations()).size().isOne(); + assertThat(deviceExportData.getInboundRelations().get(0)).matches(entityRelation -> { + return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); + }); + ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); - checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .build()); + Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); } @Test - public void testExportImportSingleDashboard_sameTenant() throws Exception { + public void testExportImportWithRelations_betweenTenants() throws Exception { logInAsTenantAdmin1(); - Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); - EntityExportData exportData = exportSingleEntity(dashboard.getId()); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); - checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .exportOutboundRelations(true) + .build()); + List> exportDataList = exportEntities(exportRequest); + + assertThat(exportDataList).allMatch(exportData -> exportData.getInboundRelations().size() + exportData.getOutboundRelations().size() == 1); + + EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); + ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); + + logInAsTenantAdmin2(); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .importOutboundRelations(true) + .build()); + Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); } - private void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { - assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); - assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); - assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); - assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); - if (initialDashboard.getAssignedCustomers() != null) { - assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); - } + @Test + public void testExportImportWithRelations_sameTenant() throws Exception { + logInAsTenantAdmin1(); + + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device1 = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); + + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(asset.getId()); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportOutboundRelations(true) + .build()); + EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); + assertThat(assetExportData.getOutboundRelations()).size().isOne(); + + Device device2 = createDevice(tenantId1, null, null, "Device 2"); + EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(assetExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .importOutboundRelations(true) + .build()); + + importEntities(importRequest); + + List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1, relation2); + } + + @Test + public void textExportImportWithRelations_sameTenant_removeExisting() throws Exception { + logInAsTenantAdmin1(); + + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); + + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(device.getId()); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .build()); + EntityExportData deviceExportData = exportEntities(exportRequest).get(0); + assertThat(deviceExportData.getInboundRelations()).size().isOne(); + + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(deviceExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .removeExistingRelations(true) + .build()); + + importEntities(importRequest); + + List relations = relationService.findByTo(TenantId.SYS_TENANT_ID, device.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + + @Test + public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(defaultDeviceProfile.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(exportRequest); + + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .findExistingByName(false) + .updateReferencesToOtherEntities(true) + .build()); + assertThatThrownBy(() -> { + importEntities(importRequest); + }).hasMessageContaining("default device profile is present"); + + importRequest.getImportSettings().setFindExistingByName(true); + importEntities(importRequest); + checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); } + @Test + public void testExportImportDashboard_betweenTenants_doNotUpdateReferencesToOtherEntities() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(customer.getId(), dashboard.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(exportRequest); + + logInAsTenantAdmin2(); + Map> importResults = importEntities(exportDataList).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { - E importedEntity = importResult.getSavedEntity(); + logInAsTenantAdmin1(); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("aaa", new TextNode("bbb"))); + dashboard = dashboardService.saveDashboard(dashboard); + dashboard = dashboardService.unassignDashboardFromCustomer(tenantId1, dashboard.getId(), customer.getId()); + EntityExportData updatedDashboardExportData = exportSingleEntity(dashboard.getId()); + assertThat(updatedDashboardExportData.getEntity().getAssignedCustomers()).isNullOrEmpty(); - assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); - assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(updatedDashboardExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(false) + .build()); + importedDashboard = (Dashboard) importEntities(importRequest).get(0).getSavedEntity(); + + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } - assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); - boolean sameTenant = tenantId1.equals(tenantId2); - if (!sameTenant) { - assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); - } else { - assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); - assertThat(importResult.getOldEntity()).isEqualTo(initialEntity); + @Test + public void testExportRequests() throws Exception { + logInAsTenantAdmin1(); + + Device device = createDevice(tenantId1, null, null, "Device 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + Customer customer = createCustomer(tenantId1, "Customer 1"); + + Map> entities = Map.of( + EntityType.DEVICE, device, EntityType.DEVICE_PROFILE, deviceProfile, + EntityType.RULE_CHAIN, ruleChain, EntityType.ASSET, asset, + EntityType.DASHBOARD, dashboard, EntityType.CUSTOMER, customer + ); + + for (ExportableEntity entity : entities.values()) { + testEntityTypeExportRequest(entity); + testCustomEntityFilterExportRequest(entity); } } + private void testEntityTypeExportRequest(ExportableEntity entity) throws Exception { + EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); + exportRequest.setEntityType(entity.getId().getEntityType()); + + List> exportDataList = exportEntities(exportRequest); + assertThat(exportDataList).size().isNotZero(); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(entity); + }); + } + + private void testCustomEntityFilterExportRequest(ExportableEntity entity) throws Exception { + CustomEntityFilterExportRequest exportRequest = new CustomEntityFilterExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); - private void logInAsTenantAdmin1() throws Exception { - login(tenantAdmin1.getEmail(), "12345678"); + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entity.getId().getEntityType()); + filter.setEntityList(List.of(entity.getId().toString())); + exportRequest.setFilter(filter); + + List> exportDataList = exportEntities(exportRequest); + assertThat(exportDataList).hasOnlyOneElementSatisfying(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(entity); + }); } - private void logInAsTenantAdmin2() throws Exception { - login(tenantAdmin2.getEmail(), "12345678"); + + @Test + public void testExportImportCustomerEntities_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + + Device tenantDevice = createDevice(tenantId1, null, null, "Tenant device 1"); + Device customerDevice = createDevice(tenantId1, customer.getId(), null, "Customer device 1"); + Asset tenantAsset = createAsset(tenantId1, null, "A", "Tenant asset 1"); + Asset customerAsset = createAsset(tenantId1, customer.getId(), "A", "Customer asset 1"); + + List exportRequests = new ArrayList<>(); + + for (EntityType entityType : Set.of(EntityType.DEVICE, EntityType.ASSET)) { + EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); + exportRequest.setEntityType(entityType); + exportRequest.setCustomerId(customer.getId()); + exportRequests.add(exportRequest); + } + + List> exportDataList = exportEntities(exportRequests); + assertThat(exportDataList).size().isEqualTo(2); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(customerDevice); + }); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(customerAsset); + }); + } + + + @Test + public void testEntityEventsOnImport() throws Exception { + logInAsTenantAdmin1(); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), deviceProfile.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + + Map entitiesExportData = exportEntities(exportRequest).stream() + .collect(Collectors.toMap(EntityExportData::getEntityType, r -> r)); + + logInAsTenantAdmin2(); + + Customer importedCustomer = (Customer) importEntity(entitiesExportData.get(EntityType.CUSTOMER)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.ADDED), isNull()); + importEntity(entitiesExportData.get(EntityType.CUSTOMER)); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + Asset importedAsset = (Asset) importEntity(entitiesExportData.get(EntityType.ASSET)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.ADDED), isNull()); + importEntity(entitiesExportData.get(EntityType.ASSET)); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + RuleChain importedRuleChain = (RuleChain) importEntity(entitiesExportData.get(EntityType.RULE_CHAIN)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + + Dashboard importedDashboard = (Dashboard) importEntity(entitiesExportData.get(EntityType.DASHBOARD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), + any(), eq(ActionType.ADDED), isNull()); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(entitiesExportData.get(EntityType.DEVICE_PROFILE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); + + ((DeviceExportData)entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); + Device importedDevice = (Device) importEntity(entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceUpdated(eq(importedDevice), isNull()); + importEntity(entitiesExportData.get(EntityType.DEVICE)); + verify(clusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index 50569b7a02..4da7de93ee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -26,4 +26,7 @@ public interface ExportableEntity extends HasId, HasName I getExternalId(); void setExternalId(I externalId); + long getCreatedTime(); + void setCreatedTime(long createdTime); + } From 119430ff9a483d12f96f410d8c93dad1ffd11d87 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 18 Apr 2022 14:32:43 +0300 Subject: [PATCH 042/178] Minor improvements --- .../sync/DefaultEntitiesExportImportService.java | 12 ++++++++++-- .../sync/importing/impl/BaseEntityImportService.java | 2 +- .../sync/importing/impl/DashboardImportService.java | 10 ++++++++++ .../sync/importing/impl/RuleChainImportService.java | 10 ++++++++++ .../server/dao/dashboard/DashboardService.java | 5 +++++ .../server/dao/rule/RuleChainService.java | 3 +++ .../server/common/data/ExportableEntity.java | 1 - .../thingsboard/server/dao/ExportableEntityDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 6 ++++++ .../server/dao/dashboard/DashboardServiceImpl.java | 7 +++++++ .../server/dao/rule/BaseRuleChainService.java | 7 ++++++- .../server/dao/sql/asset/JpaAssetDao.java | 2 +- .../server/dao/sql/customer/JpaCustomerDao.java | 2 +- .../dao/sql/dashboard/DashboardRepository.java | 3 ++- .../server/dao/sql/dashboard/JpaDashboardDao.java | 5 +++-- .../server/dao/sql/device/JpaDeviceDao.java | 2 +- .../server/dao/sql/device/JpaDeviceProfileDao.java | 2 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ----- .../server/dao/sql/rule/RuleChainRepository.java | 2 -- 19 files changed, 68 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index cfb895c36a..1db59d68f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -83,7 +83,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + fixOrder(exportDataList); List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { @@ -101,6 +101,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + private void fixOrder(List> exportDataList) { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + } + private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); @@ -137,7 +141,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findFirstByTenantIdAndName(tenantId.getId(), name); + try { + return dao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException e) { + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 97547868d2..c7aaf9c198 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -158,7 +158,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) .or(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index fda6904421..c2a3e4aea7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.HashSet; @@ -58,6 +59,15 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 5fef00cbed..dcc3b6ca92 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.Optional; @@ -50,6 +51,15 @@ public class RuleChainImportService extends BaseEntityImportService findDashboardsByTenantIdAndEdgeId(TenantId tenantId, EdgeId edgeId, PageLink pageLink); DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name); + + List findTenantDashboardsByTitle(TenantId tenantId, String title); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 4eed652bb7..9069b75710 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; +import java.util.Collection; import java.util.List; /** @@ -65,6 +66,8 @@ public interface RuleChainService { PageData findTenantRuleChainsByType(TenantId tenantId, RuleChainType type, PageLink pageLink); + Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name); + void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId); void deleteRuleChainsByTenantId(TenantId tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index 4da7de93ee..c6d13013b1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.HasId; public interface ExportableEntity extends HasId, HasName { - I getId(); void setId(I id); I getExternalId(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 467389d9d0..cd534f24e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -23,6 +23,6 @@ public interface ExportableEntityDao> extends Dao< T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - T findFirstByTenantIdAndName(UUID tenantId, String name); + default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 90353f49b3..87804fab80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -21,6 +21,9 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import java.util.List; +import java.util.UUID; + /** * The Interface DashboardDao. */ @@ -33,4 +36,7 @@ public interface DashboardDao extends Dao, TenantEntityDao, Exportabl * @return saved dashboard object */ Dashboard save(TenantId tenantId, Dashboard dashboard); + + List findByTenantIdAndTitle(UUID tenantId, String title); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index ee5c44e20f..1e82d3f27c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -40,6 +40,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -279,6 +281,11 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardInfoDao.findFirstByTenantIdAndName(tenantId.getId(), name); } + @Override + public List findTenantDashboardsByTitle(TenantId tenantId, String title) { + return dashboardDao.findByTenantIdAndTitle(tenantId.getId(), title); + } + private PaginatedRemover tenantDashboardsRemover = new PaginatedRemover() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index fd84c12a1b..8920cc0c1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -391,6 +391,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleChainDao.findRuleChainsByTenantIdAndType(tenantId.getId(), type, pageLink); } + @Override + public Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name) { + return ruleChainDao.findByTenantIdAndTypeAndName(tenantId, type, name); + } + @Override @Transactional public void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId) { @@ -457,7 +462,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ruleChain.setRoot(false); if (overwrite) { - Collection existingRuleChains = ruleChainDao.findByTenantIdAndTypeAndName(tenantId, + Collection existingRuleChains = findTenantRuleChainsByTypeAndName(tenantId, Optional.ofNullable(ruleChain.getType()).orElse(RuleChainType.CORE), ruleChain.getName()); Optional existingRuleChain = existingRuleChains.stream().findFirst(); if (existingRuleChain.isPresent()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 5988c2d2db..bcfdaae965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -215,7 +215,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im } @Override - public Asset findFirstByTenantIdAndName(UUID tenantId, String name) { + public Asset findByTenantIdAndName(UUID tenantId, String name) { return findAssetsByTenantIdAndName(tenantId, name).orElse(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index fa09304427..c6ba310793 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -76,7 +76,7 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantIdAndTitle(UUID tenantId, String title); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 771de3c813..6ff9fc6bfb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; +import java.util.List; import java.util.UUID; /** @@ -58,8 +59,8 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantIdAndTitle(UUID tenantId, String title) { + return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index d2247f5e82..93cd857660 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -309,7 +309,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao } @Override - public Device findFirstByTenantIdAndName(UUID tenantId, String name) { + public Device findByTenantIdAndName(UUID tenantId, String name) { return findDeviceByTenantIdAndName(tenantId, name).orElse(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index b21a07442f..cae94cfb73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -117,7 +117,7 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); - RuleChainEntity findFirstByTenantIdAndName(UUID tenantId, String name); - } From eacc0e6bbebd8f31dc30f9a8c129cf9b4d1f3946 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 18 Apr 2022 16:19:06 +0300 Subject: [PATCH 043/178] Remove import option whether to update references to other entities; minor improvements --- .../importing/data/EntityImportSettings.java | 2 - .../importing/impl/AssetImportService.java | 6 +-- .../impl/BaseEntityImportService.java | 43 +++-------------- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 23 +++++---- .../importing/impl/DeviceImportService.java | 12 ++--- .../impl/DeviceProfileImportService.java | 12 ++--- .../impl/RuleChainImportService.java | 8 ++-- ...aseEntitiesExportImportControllerTest.java | 4 +- ...EntitiesExportImportControllerSqlTest.java | 47 +------------------ 10 files changed, 42 insertions(+), 119 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 3b457e1f5a..d4b0c46f31 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -30,6 +30,4 @@ public class EntityImportSettings { private boolean importInboundRelations; private boolean importOutboundRelations; private boolean removeExistingRelations; - - private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 02eaae27b5..ae54a988f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -36,13 +36,13 @@ public class AssetImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData exportData, IdProvider idProvider) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index c7aaf9c198..82df075238 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -40,13 +40,9 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Function; import java.util.stream.Collectors; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -68,7 +64,7 @@ public abstract class BaseEntityImportService importResult, D exportData, - NewIdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { + IdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); @@ -179,37 +175,10 @@ public abstract class BaseEntityImportService ID get(Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return getInternalId(idExtractor.apply(this.entity)); - } else { - return idExtractor.apply(existingEntity); - } - } - - public ID getInternal(ID externalId) { - return getInternalId(externalId); - } - - public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() - .peek(t -> { - idSetter.accept(t, getInternalId(idGetter.apply(t))); - }) - .collect(Collectors.toSet()); - } else { - return listExtractor.apply(existingEntity); - } - } - private ID getInternalId(ID externalId) { + public ID getInternalId(ID externalId) { if (externalId == null || externalId.isNullUid()) return null; HasId entity = findInternalEntity(user.getTenantId(), externalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index b03c0ab335..1a79fe666b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -36,12 +36,12 @@ public class CustomerImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData exportData, IdProvider idProvider) { return customerService.saveCustomer(customer); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index c2a3e4aea7..e7c5ed0ea3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -55,7 +55,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) .filter(JsonNode::isObject) @@ -84,15 +84,17 @@ public class DashboardImportService extends BaseEntityImportService { String uuid = matchResult.group(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.getInternal(entityId).toString(); + return idProvider.getInternalId(entityId).toString(); }); ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); }); })); - // TODO [viacheslav]: improve the code below + Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() + .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) + .collect(Collectors.toSet()); + if (dashboard.getId() == null) { - Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); dashboard = dashboardService.saveDashboard(dashboard); for (ShortCustomerInfo customerInfo : assignedCustomers) { @@ -101,20 +103,21 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - Set newAssignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() - .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = assignedCustomers.stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set toUnassign = new HashSet<>(existingAssignedCustomers); toUnassign.removeAll(newAssignedCustomers); for (CustomerId customerId : toUnassign) { - dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId); + assignedCustomers = dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); } + Set toAssign = new HashSet<>(newAssignedCustomers); toAssign.removeAll(existingAssignedCustomers); for (CustomerId customerId : toAssign) { - dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + assignedCustomers = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); } - dashboard.setAssignedCustomers(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()); + + dashboard.setAssignedCustomers(assignedCustomers); dashboard = dashboardService.saveDashboard(dashboard); } return dashboard; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 504bd1a828..091b6eb856 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -35,16 +35,16 @@ public class DeviceImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { - deviceProfile.setDefaultRuleChainId(idProvider.get(DeviceProfile::getDefaultRuleChainId)); - deviceProfile.setDefaultDashboardId(idProvider.get(DeviceProfile::getDefaultDashboardId)); - deviceProfile.setFirmwareId(idProvider.get(DeviceProfile::getFirmwareId)); - deviceProfile.setSoftwareId(idProvider.get(DeviceProfile::getSoftwareId)); + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); + deviceProfile.setSoftwareId(idProvider.getInternalId(deviceProfile.getSoftwareId())); return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index dcc3b6ca92..1e4ed958a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -47,7 +47,7 @@ public class RuleChainImportService extends BaseEntityImportService { @@ -73,14 +73,14 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.getInternal(new RuleChainId(otherRuleChainUuid)).toString() + idProvider.getInternalId(new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternal(ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index a8a5bca534..b5c5a5aab8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -360,9 +360,7 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected List> importEntities(List> exportDataList) throws Exception { ImportRequest importRequest = new ImportRequest(); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(true) - .build()); + importRequest.setImportSettings(new EntityImportSettings()); importRequest.setExportDataList(exportDataList); return importEntities(importRequest); } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 920d91e93d..8a0c958b2c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -376,9 +376,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); ImportRequest importRequest = new ImportRequest(); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(true) - .build()); + importRequest.setImportSettings(new EntityImportSettings()); importRequest.setExportDataList(exportDataList); Map> importResults = importEntities(importRequest).stream() .collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -583,7 +581,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() .findExistingByName(false) - .updateReferencesToOtherEntities(true) .build()); assertThatThrownBy(() -> { importEntities(importRequest); @@ -594,48 +591,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); } - @Test - public void testExportImportDashboard_betweenTenants_doNotUpdateReferencesToOtherEntities() throws Exception { - logInAsTenantAdmin1(); - Customer customer = createCustomer(tenantId1, "Customer 1"); - Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1"); - - EntityListExportRequest exportRequest = new EntityListExportRequest(); - exportRequest.setEntitiesIds(List.of(customer.getId(), dashboard.getId())); - exportRequest.setExportSettings(new EntityExportSettings()); - List> exportDataList = exportEntities(exportRequest); - - logInAsTenantAdmin2(); - Map> importResults = importEntities(exportDataList).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); - - Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); - Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); - assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { - assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); - }); - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - - logInAsTenantAdmin1(); - dashboard.setConfiguration(JacksonUtil.newObjectNode().set("aaa", new TextNode("bbb"))); - dashboard = dashboardService.saveDashboard(dashboard); - dashboard = dashboardService.unassignDashboardFromCustomer(tenantId1, dashboard.getId(), customer.getId()); - EntityExportData updatedDashboardExportData = exportSingleEntity(dashboard.getId()); - assertThat(updatedDashboardExportData.getEntity().getAssignedCustomers()).isNullOrEmpty(); - - logInAsTenantAdmin2(); - ImportRequest importRequest = new ImportRequest(); - importRequest.setExportDataList(List.of(updatedDashboardExportData)); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(false) - .build()); - importedDashboard = (Dashboard) importEntities(importRequest).get(0).getSavedEntity(); - - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { - assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); - }); - } - @Test public void testExportRequests() throws Exception { From a5cdbca690b431bca9c7ad99a585840f50edac99 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 13:37:45 +0300 Subject: [PATCH 044/178] Relations import: log entity action --- .../impl/BaseEntityImportService.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 82df075238..5802e140f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -91,11 +91,16 @@ public abstract class BaseEntityImportService importResult, D exportData, IdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); + importResult.addSendEventsCallback(() -> { + onEntitySaved(user, savedEntity, oldEntity); + }); + importResult.addSaveReferencesCallback(() -> { List newRelations = new LinkedList<>(); @@ -107,7 +112,7 @@ public abstract class BaseEntityImportService { + entityActionService.logEntityAction(user, relation.getFrom(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityActionService.logEntityAction(user, relation.getTo(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + }); } }); + } + private void deleteRelation(SecurityUser user, EntityRelation relation, EntityImportResult importResult) { + relationService.deleteRelation(user.getTenantId(), relation); importResult.addSendEventsCallback(() -> { - onEntitySaved(user, savedEntity, oldEntity); + entityActionService.logEntityAction(user, relation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, relation); + entityActionService.logEntityAction(user, relation.getTo(), null, null, + ActionType.RELATION_DELETED, null, relation); }); } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), From 832f32fbe1e5cfb28c2a47f30e9210fd62b9ece7 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 14:34:55 +0300 Subject: [PATCH 045/178] Improvements for relations import --- .../sync/exporting/data/EntityExportData.java | 3 +- .../impl/DefaultEntityExportService.java | 22 ++--- .../importing/data/EntityImportSettings.java | 5 +- .../impl/BaseEntityImportService.java | 89 +++++++++---------- ...EntitiesExportImportControllerSqlTest.java | 27 +++--- 5 files changed, 68 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index 4ab6dd57b1..728d8212c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -43,7 +43,6 @@ public class EntityExportData> { private E entity; private EntityType entityType; - private List inboundRelations; - private List outboundRelations; + private List relations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java index 4fe09e9322..672bc50109 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import java.util.ArrayList; import java.util.List; @Service @@ -63,24 +64,25 @@ public class DefaultEntityExportService relations = null; if (exportSettings.isExportInboundRelations()) { List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (inboundRelations != null) { - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); } - exportData.setInboundRelations(inboundRelations); + relations = new ArrayList<>(inboundRelations); } if (exportSettings.isExportOutboundRelations()) { List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (outboundRelations != null) { - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } - exportData.setOutboundRelations(outboundRelations); + if (relations == null) { + relations = new ArrayList<>(); + } + relations.addAll(outboundRelations); } + exportData.setRelations(relations); } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index d4b0c46f31..a5de7d8744 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -26,8 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean findExistingByName; - - private boolean importInboundRelations; - private boolean importOutboundRelations; - private boolean removeExistingRelations; + private boolean updateRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 5802e140f3..1edd6a1ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -40,10 +39,9 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -102,47 +100,53 @@ public abstract class BaseEntityImportService { - List newRelations = new LinkedList<>(); - - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> relation.setTo(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); - deleteRelation(user, existingRelation, importResult); - } - } - } - if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { - newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> relation.setFrom(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); - deleteRelation(user, existingRelation, importResult); - } - } + List relations = exportData.getRelations(); + if (relations == null || !importSettings.isUpdateRelations()) { + return; } + relations = new ArrayList<>(relations); - for (EntityRelation relation : newRelations) { - HasId otherEntity = null; + for (EntityRelation relation : relations) { if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(otherEntity.getId()); + HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); + exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); + relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(otherEntity.getId()); + if (!relation.getFrom().equals(savedEntity.getId())){ + HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); + exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); + relation.setFrom(from.getId()); } - if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); + } + + if (oldEntity != null) { + List existingRelations = new ArrayList<>(); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + + for (EntityRelation existingRelation : existingRelations) { + if (!relations.contains(existingRelation)) { + EntityId otherEntity = null; + if (!existingRelation.getTo().equals(savedEntity.getId())) { + otherEntity = existingRelation.getTo(); + } else if (!existingRelation.getFrom().equals(savedEntity.getId())){ + otherEntity = existingRelation.getFrom(); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + } + relationService.deleteRelation(user.getTenantId(), existingRelation); + importResult.addSendEventsCallback(() -> { + entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + entityActionService.logEntityAction(user, existingRelation.getTo(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + }); + } } + } + for (EntityRelation relation : relations) { relationService.saveRelation(user.getTenantId(), relation); importResult.addSendEventsCallback(() -> { entityActionService.logEntityAction(user, relation.getFrom(), null, null, @@ -154,17 +158,6 @@ public abstract class BaseEntityImportService importResult) { - relationService.deleteRelation(user.getTenantId(), relation); - importResult.addSendEventsCallback(() -> { - entityActionService.logEntityAction(user, relation.getFrom(), null, null, - ActionType.RELATION_DELETED, null, relation); - entityActionService.logEntityAction(user, relation.getTo(), null, null, - ActionType.RELATION_DELETED, null, relation); - }); - } - - protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 8a0c958b2c..8d1857f133 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -311,7 +311,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).get(0).getSavedEntity(); Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() - .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) .map(JsonNode::asText).collect(Collectors.toSet()); assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); @@ -427,8 +427,8 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp List> exportDataList = exportEntities(exportRequest); EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); - assertThat(deviceExportData.getInboundRelations()).size().isOne(); - assertThat(deviceExportData.getInboundRelations().get(0)).matches(entityRelation -> { + assertThat(deviceExportData.getRelations()).size().isOne(); + assertThat(deviceExportData.getRelations().get(0)).matches(entityRelation -> { return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); }); ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); @@ -439,7 +439,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) + .updateRelations(true) .build()); Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -473,7 +473,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .build()); List> exportDataList = exportEntities(exportRequest); - assertThat(exportDataList).allMatch(exportData -> exportData.getInboundRelations().size() + exportData.getOutboundRelations().size() == 1); + assertThat(exportDataList).allMatch(exportData -> exportData.getRelations().size() == 1); EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); @@ -484,8 +484,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) - .importOutboundRelations(true) + .updateRelations(true) .build()); Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -515,7 +514,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .exportOutboundRelations(true) .build()); EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); - assertThat(assetExportData.getOutboundRelations()).size().isOne(); + assertThat(assetExportData.getRelations()).size().isOne(); Device device2 = createDevice(tenantId1, null, null, "Device 2"); EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); @@ -523,13 +522,14 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(List.of(assetExportData)); importRequest.setImportSettings(EntityImportSettings.builder() - .importOutboundRelations(true) + .updateRelations(true) .build()); importEntities(importRequest); List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); - assertThat(relations).contains(relation1, relation2); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); } @Test @@ -546,7 +546,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .exportInboundRelations(true) .build()); EntityExportData deviceExportData = exportEntities(exportRequest).get(0); - assertThat(deviceExportData.getInboundRelations()).size().isOne(); + assertThat(deviceExportData.getRelations()).size().isOne(); Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); @@ -554,8 +554,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(List.of(deviceExportData)); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) - .removeExistingRelations(true) + .updateRelations(true) .build()); importEntities(importRequest); @@ -730,7 +729,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); - ((DeviceExportData)entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); + ((DeviceExportData) entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); Device importedDevice = (Device) importEntity(entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), any(), eq(ActionType.ADDED), isNull()); From 457d2340a905ec5d98346ac56618cdf5d9f46044 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 16:23:54 +0300 Subject: [PATCH 046/178] Refactor relations export --- .../data/request/EntityExportSettings.java | 3 +-- .../impl/DefaultEntityExportService.java | 16 +++++++--------- .../EntitiesExportImportControllerSqlTest.java | 10 ++++------ 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index ddf2e43bcf..5454105ee5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -25,6 +25,5 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityExportSettings { - private boolean exportInboundRelations; - private boolean exportOutboundRelations; + private boolean exportRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java index 672bc50109..a051f0e59f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -64,25 +64,23 @@ public class DefaultEntityExportService relations = null; - if (exportSettings.isExportInboundRelations()) { + if (exportSettings.isExportRelations()) { + List relations = new ArrayList<>(); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); for (EntityRelation relation : inboundRelations) { exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); } - relations = new ArrayList<>(inboundRelations); - } - if (exportSettings.isExportOutboundRelations()) { + relations.addAll(inboundRelations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); for (EntityRelation relation : outboundRelations) { exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } - if (relations == null) { - relations = new ArrayList<>(); - } relations.addAll(outboundRelations); + + exportData.setRelations(relations); } - exportData.setRelations(relations); } protected D newExportData() { diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 8d1857f133..44d8f5f461 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -421,8 +421,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityListExportRequest exportRequest = new EntityListExportRequest(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) - .exportOutboundRelations(false) + .exportRelations(true) .build()); List> exportDataList = exportEntities(exportRequest); @@ -468,8 +467,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityListExportRequest exportRequest = new EntityListExportRequest(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) - .exportOutboundRelations(true) + .exportRelations(true) .build()); List> exportDataList = exportEntities(exportRequest); @@ -511,7 +509,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); exportRequest.setEntityId(asset.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportOutboundRelations(true) + .exportRelations(true) .build()); EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); assertThat(assetExportData.getRelations()).size().isOne(); @@ -543,7 +541,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); exportRequest.setEntityId(device.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) + .exportRelations(true) .build()); EntityExportData deviceExportData = exportEntities(exportRequest).get(0); assertThat(deviceExportData.getRelations()).size().isOne(); From 428ff2851dd3fbf189b58b64baea369b13b8297f Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 18:25:56 +0300 Subject: [PATCH 047/178] Minor refactoring --- .../sync/DefaultEntitiesExportImportService.java | 2 +- .../sync/exporting/impl/DeviceExportService.java | 2 +- .../sync/importing/impl/BaseEntityImportService.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 1db59d68f1..44a9396fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -153,7 +153,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { if (entity instanceof HasTenantId) { accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); - } else if (entity != null) { + } else { accessControlService.checkPermission(user, Resource.of(entityType), operation); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index da1e4f3a96..21df8beda2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -36,7 +36,7 @@ public class DeviceExportService extends BaseEntityExportService { - List relations = exportData.getRelations(); - if (relations == null || !importSettings.isUpdateRelations()) { + if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { return; } - relations = new ArrayList<>(relations); + + List relations = new ArrayList<>(exportData.getRelations()); for (EntityRelation relation : relations) { if (!relation.getTo().equals(savedEntity.getId())) { @@ -112,7 +112,7 @@ public abstract class BaseEntityImportService from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); @@ -129,7 +129,7 @@ public abstract class BaseEntityImportService Date: Thu, 21 Apr 2022 19:13:58 +0300 Subject: [PATCH 048/178] Refactor export/import api --- .../EntitiesExportImportController.java | 104 ++------- .../DefaultEntitiesExportImportService.java | 88 +------- .../DefaultExportableEntitiesService.java | 206 ++++++++++++++++++ .../exporting/ExportableEntitiesService.java | 5 + .../sync/exporting/data/DeviceExportData.java | 2 + .../exporting/data/RuleChainExportData.java | 2 + 6 files changed, 239 insertions(+), 168 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 4a1badd38e..3dff560ac1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,47 +17,25 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.dao.DataAccessException; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.query.EntityData; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @RestController @RequestMapping("/api/entities") @@ -67,7 +45,8 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final EntityService entityService; + private final ExportableEntitiesService exportableEntitiesService; + @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -76,6 +55,7 @@ public class EntitiesExportImportController extends BaseController { try { return exportEntitiesByRequest(user, exportRequest); } catch (Exception e) { + log.warn("Failed to export entities for request {}", exportRequest, e); throw handleException(e); } } @@ -85,84 +65,29 @@ public class EntitiesExportImportController extends BaseController { public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List> exportDataList = new ArrayList<>(); + List> result = new ArrayList<>(); for (ExportRequest exportRequest : exportRequests) { - exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); + List> exportDataList = exportEntitiesByRequest(user, exportRequest); + result.addAll(exportDataList); } - return exportDataList; + return result; } catch (Exception e) { + log.warn("Failed to export entities for requests {}", exportRequests, e); throw handleException(e); } } - - private List> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { - List entitiesIds = findEntitiesForRequest(user, request); + private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { + List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); List> exportDataList = new ArrayList<>(); - for (EntityId entityId : entitiesIds) { - exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); + for (EntityId entityId : entities) { + EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportRequest.getExportSettings()); + exportDataList.add(exportData); } return exportDataList; } - private List findEntitiesForRequest(SecurityUser user, ExportRequest request) { - switch (request.getType()) { - case SINGLE_ENTITY: { - return List.of(((SingleEntityExportRequest) request).getEntityId()); - } - case ENTITY_LIST: { - return ((EntityListExportRequest) request).getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(exportRequest.getEntityType()); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); - } - case CUSTOM_ENTITY_FILTER: { - CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; - EntityFilter filter = exportRequest.getFilter(); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); - } - case CUSTOM_ENTITY_QUERY: { - CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; - EntityDataQuery query = exportRequest.getQuery(); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByQuery(user.getTenantId(), customerId, query); - } - default: - throw new IllegalArgumentException("Export request is not supported"); - } - } - - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return findEntitiesByQuery(tenantId, customerId, query); - } - - private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { - try { - return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() - .map(EntityData::getEntityId) - .collect(Collectors.toList()); - } catch (DataAccessException e) { - log.error("Failed to find entity data by query: {}", e.getMessage()); - throw new IllegalArgumentException("Entity filter cannot be processed"); - } - } - @PostMapping("/import") public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { @@ -183,6 +108,7 @@ public class EntitiesExportImportController extends BaseController { return importResults; } catch (Exception e) { + log.warn("Failed to import entities for request {}", importRequest, e); throw handleException(e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 44a9396fab..75c4ff9e12 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,26 +16,18 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.AccessControlService; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; @@ -57,13 +49,11 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { +@Slf4j +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); - - private final AccessControlService accessControlService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -117,75 +107,24 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Override - public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { - EntityType entityType = externalId.getEntityType(); - if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { - ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - } else { - return null; - } - } - - @Override - public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { - Dao dao = (Dao) getDao(id.getEntityType()); - E entity = dao.findById(tenantId, id.getId()); - if (entity instanceof HasTenantId && !((HasTenantId) entity).getTenantId().equals(tenantId)) { - return null; - } - return entity; - } - - @Override - public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { - ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - try { - return dao.findByTenantIdAndName(tenantId.getId(), name); - } catch (UnsupportedOperationException e) { - return null; - } - } - - - @Override - public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { - if (entity instanceof HasTenantId) { - accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); - } else { - accessControlService.checkPermission(user, Resource.of(entityType), operation); - } - } - - @Override - public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { - HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); - checkPermission(user, entity, entityId.getEntityType(), operation); - } - - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { - if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + EntityExportService exportService = exportServices.get(entityType); + if (exportService == null) { throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); } - return (EntityExportService) exportServices.get(entityType); + return (EntityExportService) exportService; } @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { - if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + EntityImportService importService = importServices.get(entityType); + if (importService == null) { throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); } - return (EntityImportService) importServices.get(entityType); + return (EntityImportService) importService; } - private Dao getDao(EntityType entityType) { - return daos.get(entityType); - } - - @Autowired private void setExportServices(DefaultEntityExportService defaultExportService, Collection> exportServices) { @@ -208,13 +147,4 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS }); } - @Autowired - private void setDaos(Collection> daos) { - daos.forEach(dao -> { - if (dao.getEntityType() != null) { - this.daos.put(dao.getEntityType(), dao); - } - }); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java new file mode 100644 index 0000000000..c2c99c63f5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -0,0 +1,206 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataAccessException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultExportableEntitiesService implements ExportableEntitiesService { + + private final Map> daos = new HashMap<>(); + + private final EntityService entityService; + private final AccessControlService accessControlService; + + + @Override + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { + EntityType entityType = externalId.getEntityType(); + Dao dao = getDao(entityType); + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + return exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } else { + return null; + } + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + + E entity = dao.findById(tenantId, id.getId()); + if (((HasTenantId) entity).getTenantId().equals(tenantId)) { + return entity; + } + return null; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + Dao dao = getDao(entityType); + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + try { + return exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException ignored) { + } + } + return null; + } + + + @Override + public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { + switch (request.getType()) { + case SINGLE_ENTITY: { + return List.of(((SingleEntityExportRequest) request).getEntityId()); + } + case ENTITY_LIST: { + return ((EntityListExportRequest) request).getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(exportRequest.getEntityType()); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case CUSTOM_ENTITY_FILTER: { + CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; + EntityFilter filter = exportRequest.getFilter(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case CUSTOM_ENTITY_QUERY: { + CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; + EntityDataQuery query = exportRequest.getQuery(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByQuery(tenantId, customerId, query); + } + default: { + throw new IllegalArgumentException("Export request is not supported"); + } + } + } + + private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + return findEntitiesByQuery(tenantId, customerId, query); + } + + private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + try { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + } catch (DataAccessException e) { + log.error("Failed to find entity data by query: {}", e.getMessage()); + throw new IllegalArgumentException("Entity filter cannot be processed"); + } + } + + + @Override + public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { + if (entity instanceof HasTenantId) { + accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); + } else { + accessControlService.checkPermission(user, Resource.of(entityType), operation); + } + } + + @Override + public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { + HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); + checkPermission(user, entity, entityId.getEntityType(), operation); + } + + + @SuppressWarnings("unchecked") + private Dao getDao(EntityType entityType) { + return (Dao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 5cf31042e0..9021a03e85 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; + +import java.util.List; public interface ExportableEntitiesService { @@ -32,6 +35,8 @@ public interface ExportableEntitiesService { , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); + List findEntitiesForRequest(TenantId tenantId, ExportRequest request); + void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 30d84c0a44..56633f9c91 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -17,10 +17,12 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) @Data public class DeviceExportData extends EntityExportData { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index 025459f27e..2e05e2f1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -17,10 +17,12 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) @Data public class RuleChainExportData extends EntityExportData { From 7e2910ed9549bf5fe6973530293fc46bd751c2d2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 25 Apr 2022 14:28:54 +0300 Subject: [PATCH 049/178] Minor improvements --- .../EntitiesExportImportController.java | 20 +++++++++- .../DefaultExportableEntitiesService.java | 39 +++++++++++++------ .../CustomEntityFilterExportRequest.java | 5 ++- .../CustomEntityQueryExportRequest.java | 5 ++- .../data/request/EntityTypeExportRequest.java | 5 ++- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 3dff560ac1..afe2a4ac1f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -29,8 +29,10 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; @@ -79,10 +81,16 @@ public class EntitiesExportImportController extends BaseController { private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); + EntityExportSettings exportSettings = exportRequest.getExportSettings(); + if (exportSettings == null) { + exportSettings = EntityExportSettings.builder() + .exportRelations(false) + .build(); + } List> exportDataList = new ArrayList<>(); for (EntityId entityId : entities) { - EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportRequest.getExportSettings()); + EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportSettings); exportDataList.add(exportData); } return exportDataList; @@ -93,7 +101,15 @@ public class EntitiesExportImportController extends BaseController { public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + EntityImportSettings importSettings = importRequest.getImportSettings(); + if (importSettings == null) { + importSettings = EntityImportSettings.builder() + .findExistingByName(false) + .updateRelations(false) + .build(); + } + + List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importSettings); importResults.stream() .map(EntityImportResult::getSendEventsCallback) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index c2c99c63f5..a616dec6c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; @@ -56,7 +57,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @@ -78,12 +78,17 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi EntityType entityType = externalId.getEntityType(); Dao dao = getDao(entityType); + E entity = null; + if (dao instanceof ExportableEntityDao) { ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; - return exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - } else { + entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } + if (entity == null || !belongsToTenant(entity, tenantId)) { return null; } + + return entity; } @Override @@ -92,24 +97,36 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi Dao dao = getDao(entityType); E entity = dao.findById(tenantId, id.getId()); - if (((HasTenantId) entity).getTenantId().equals(tenantId)) { - return entity; + + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; } - return null; + + return entity; } @Override public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { Dao dao = getDao(entityType); + E entity = null; + if (dao instanceof ExportableEntityDao) { ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; try { - return exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); } catch (UnsupportedOperationException ignored) { } } - return null; + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + private boolean belongsToTenant(HasId entity, TenantId tenantId) { + return tenantId.equals(((HasTenantId) entity).getTenantId()); } @@ -127,21 +144,21 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); entityTypeFilter.setEntityType(exportRequest.getEntityType()); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); } case CUSTOM_ENTITY_FILTER: { CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; EntityFilter filter = exportRequest.getFilter(); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); } case CUSTOM_ENTITY_QUERY: { CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; EntityDataQuery query = exportRequest.getQuery(); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByQuery(tenantId, customerId, query); } default: { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java index 02cbca98d6..d71ddc8720 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java @@ -17,9 +17,10 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.query.EntityFilter; +import java.util.UUID; + @EqualsAndHashCode(callSuper = true) @Data public class CustomEntityFilterExportRequest extends ExportRequest { @@ -27,7 +28,7 @@ public class CustomEntityFilterExportRequest extends ExportRequest { private EntityFilter filter; private int page; private int pageSize; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java index 76d258e891..946676cf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java @@ -17,15 +17,16 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.UUID; + @EqualsAndHashCode(callSuper = true) @Data public class CustomEntityQueryExportRequest extends ExportRequest { private EntityDataQuery query; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java index 82cb023258..fd2f659b70 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java @@ -18,7 +18,8 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; @EqualsAndHashCode(callSuper = true) @Data @@ -27,7 +28,7 @@ public class EntityTypeExportRequest extends ExportRequest { private EntityType entityType; private int page; private int pageSize; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { From 5736e52d3e38a5ffaff6c7f0a6a91ebdba22c788 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 25 Apr 2022 14:54:53 +0300 Subject: [PATCH 050/178] Fix EntitiesExportImportControllerSqlTest --- .../server/controller/EntitiesExportImportController.java | 1 + .../controller/sql/EntitiesExportImportControllerSqlTest.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index afe2a4ac1f..a6ad9f1ca6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -81,6 +81,7 @@ public class EntitiesExportImportController extends BaseController { private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); + EntityExportSettings exportSettings = exportRequest.getExportSettings(); if (exportSettings == null) { exportSettings = EntityExportSettings.builder() diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 44d8f5f461..bd237e39d0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -659,7 +659,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entityType); - exportRequest.setCustomerId(customer.getId()); + exportRequest.setCustomerId(customer.getUuidId()); exportRequests.add(exportRequest); } From ba1ea3521034b2eb9a6cb24c5165bf88396480be Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 26 Apr 2022 12:08:05 +0300 Subject: [PATCH 051/178] Swagger docs for export-import controller --- .../EntitiesExportImportController.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a6ad9f1ca6..c91fd0c292 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,6 +40,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; + @RestController @RequestMapping("/api/entities") @TbCoreComponent @@ -50,6 +53,42 @@ public class EntitiesExportImportController extends BaseController { private final ExportableEntitiesService exportableEntitiesService; + @ApiOperation(value = "Export entities by request", notes = "" + + "Takes export request and returns list of export data for each entity found by request. " + + "Supported entity types for export, hence for import, are **DEVICE**, **DEVICE_PROFILE**, **ASSET**, " + + "**CUSTOMER**, **RULE_CHAIN** and **DASHBOARD**." + NEW_LINE + + "For each type of export request, you can set some export settings: \n" + + "- **exportRelations** - whether to export inbound and outbound relations for an entity " + + "(only relations of type group COMMON can be exported)" + NEW_LINE + + "Supported export requests:\n" + + "- **SINGLE_ENTITY**:" + NEW_LINE + + " To export a single entity by id. Example:" + NEW_LINE + + "```\n{\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "- **ENTITY_LIST**:" + NEW_LINE + + " To export a list of entities by their ids. Example:" + NEW_LINE + + "```\n{\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"ASSET\",\n \"id\": \"2f0a3bd0-989d-11ec-93b5-6de6c2b68078\"\n }\n ],\n \"exportSettings\": {\n \"exportRelations\": true\n }\n}\n```" + NEW_LINE + + "- **ENTITY_TYPE**:" + NEW_LINE + + " To export entities of specified entity type. You need to specify page size, " + + "and may specify page index and customer id (to limit the list of entities to the ones owned by a customer). " + + "Entities are ordered by created time descendingly. Example:" + NEW_LINE + + "```\n{\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n}\n```" + NEW_LINE + + "- **CUSTOM_ENTITY_FILTER**:" + NEW_LINE + + " To export entities by custom entity filter. The order used is the same as for ENTITY_TYPE export request. Example:" + NEW_LINE + + "```\n{\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"Thermostats\",\n \"deviceNameFilter\": \"\"\n },\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "- **CUSTOM_ENTITY_QUERY**:" + NEW_LINE + + " To export entities by custom entity query. Example: " + NEW_LINE + + "```\n{\n \"type\": \"CUSTOM_ENTITY_QUERY\",\n \"query\": {\n \"entityFilter\": {\n \"type\": \"entityType\",\n \"entityType\": \"DEVICE\"\n },\n \"pageLink\": {\n \"page\": 0,\n \"pageSize\": 200,\n \"textSearch\": \"THB_\",\n \"sortOrder\": {\n \"key\": {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n },\n \"direction\": \"DESC\"\n }\n },\n \"entityFields\": [\n {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n }\n ],\n \"latestValues\": [\n {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n }\n ],\n \"keyFilters\": [\n {\n \"key\": {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n },\n \"valueType\": \"NUMERIC\",\n \"predicate\": {\n \"type\": \"NUMERIC\",\n \"operation\": \"GREATER\",\n \"value\": {\n \"defaultValue\": 0\n }\n }\n }\n ]\n },\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "Mostly, export data of an entity contains the whole entity itself and its relations " + + "(if option to export relations was enabled):" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"ASSET\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1648204424029,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": { ... },\n \"customerId\": { ... },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n ...\n },\n \"relations\": [\n {\n \"from\": {\n \"entityType\": \"ASSET\",\n \"id\": ...\n },\n \"to\": {\n \"entityType\": \"DEVICE\",\n \"id\": ...\n },\n \"type\": \"Contains\",\n \"typeGroup\": \"COMMON\",\n \"additionalInfo\": {\n \"a\": \"b\"\n }\n }\n ]\n }\n]\n```" + NEW_LINE + + "For devices, export data will additionally contain device's credentials; for rule chains - its metadata:" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"DEVICE\",\n \"entity\": { ... },\n \"credentials\": {\n \"id\": { ... },\n \"createdTime\": 1648829321209,\n \"deviceId\": { ... },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"5cZEDo45KGW7JgVNv4Ko\",\n \"credentialsValue\": null\n }\n }\n]\n```" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"RULE_CHAIN\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1646056614257,\n \"additionalInfo\": null,\n \"tenantId\": { ... },\n \"name\": \"Rule Chain 2\",\n \"type\": \"CORE\",\n \"firstRuleNodeId\": { ... },\n \"root\": false,\n ...\n },\n \"metaData\": {\n \"ruleChainId\": { ... },\n \"firstNodeIndex\": 7,\n \"nodes\": [ ... ],\n \"connections\": [ ... ],\n \"ruleChainConnections\": null\n }\n }\n]\n```" + NEW_LINE + + "Returned export data is to be used later for import request." + NEW_LINE + + "If any entity found by request is of unsupported type - an error will be returned.\n" + + "Also, if a user does not have a READ permission for an entity (or, if relations are exported, for a bounded entity), " + + "access will be denied." + + ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { @@ -62,6 +101,12 @@ public class EntitiesExportImportController extends BaseController { } } + @ApiOperation(value = "Export entities by multiple requests", notes = "" + + "The API behaviour is the same as for exporting entities by single request, " + + "except that this method takes an array of export requests as a request body." + NEW_LINE + + "Example:" + NEW_LINE + + "```\n[\n {\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"5f9eda10-b442-11ec-bbf5-adec34031568\"\n }\n },\n {\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"thermostat\",\n \"deviceNameFilter\": \"\"\n },\n \"pageSize\": 1000\n },\n {\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"pageSize\": 1000,\n \"exportSettings\": {\n \"exportRelations\": true\n }\n },\n {\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"2ef13590-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"e7311ec0-b442-11ec-bbf5-adec34031568\"\n }\n ]\n }\n]\n```" + + ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) @PostMapping(value = "/export", params = {"multiple"}) @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { @@ -98,6 +143,34 @@ public class EntitiesExportImportController extends BaseController { } + @ApiOperation(value = "Import entities by request", notes = "" + + "Takes import request and returns the list of import results. " + + "Import request must contain the list of export data and might contain import settings. " + NEW_LINE + + "The method creates an entity if it is new in the scope of a tenant, or otherwise updates an existing one. " + + "On entity import request, we first try to find an entity within a tenant that has externalId equal " + + "to the id in the export data. If the platform fails to do that, we then search for an entity with " + + "regular (internal) id like the one in the export data (this is useful in case we are exporting and " + + "importing entities within the same tenant). Then, if we still haven't found any entity, if findExistingByName " + + "option of the EntityImportSettings is enabled, we will search for the one by its name (this is also useful " + + "for avoiding conflicts with default device profile or Root Rule Chain when importing all entities from another " + + "tenant). After, if the exported entity is new for this tenant, we simply save it with external id " + + "from the export data, and also create relations if any (if updateRelations option is enabled). " + + "Otherwise, we will reset all fields of the existing entity to the ones from the export data and save it, " + + "and also will update the list of relations (remove the ones that aren't present in the export data, " + + "and update or create others), if updateRelations option is enabled." + NEW_LINE + + "If an entity contains references to some other entities, like device references certain device profile, " + + "we will find this other entity within the tenant by this principle: look for an entity with " + + "such external id, or otherwise, internal id. This requires referenced entities to be imported " + + "before the referencing entity (if we are importing to another tenant). So, when receiving the list " + + "of entities' export data for import, we first try to fix the data order to import 'standalone' entities first." + NEW_LINE + + "As for relations importing, they are processed after all entities in the import batch are already saved, " + + "and the internal id of a bounded entity is found with the regular principle." + NEW_LINE + + "Import of all entities and their relations from the import request is processed in the single transaction, " + + "and so everything will be rolled back if the platform fails to e.g. find internal entity by external id. \n" + + "Example of import request:\n" + + "```\n{\n \"importSettings\": {\n \"findExistingByName\": false,\n \"updateRelations\": false\n },\n \"exportDataList\": [\n {\n \"entityType\": \"DEVICE_PROFILE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n \"createdTime\": 1649096026765,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"name\": \"Profile 1\",\n ...\n }\n },\n {\n \"entityType\": \"DEVICE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154276962,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"customerId\": {\n \"entityType\": \"CUSTOMER\",\n \"id\": \"13814000-1dd2-11b2-8080-808080808080\"\n },\n \"name\": \"Device 1\",\n \"type\": \"Profile 1\",\n \"label\": \"v1.0\",\n \"deviceProfileId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n ...\n },\n \"credentials\": {\n \"id\": {\n \"id\": \"981e0360-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154277014,\n \"deviceId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"sGExNdnl71uKmkNvtNdp\",\n \"credentialsValue\": null\n }\n }\n ]\n}\n```" + NEW_LINE + + "The response contains a list of EntityImportResult which has values of savedEntity and oldEntity:\n" + + "```\n[\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v2.0\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"6b03ab20-989e-11ec-b446-89df822d7fa2\"\n }\n },\n \"oldEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v1.0\",\n ...\n \"externalId\": null\n },\n \"entityType\": \"ASSET\"\n },\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"387213f0-b4ea-11ec-abfc-6dcc6508d0b5\"\n },\n \"createdTime\": 1649167860399,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 2\",\n \"type\": \"B\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"0b9ea0d0-ac27-11ec-a2a6-89d15eae3b21\"\n }\n },\n \"oldEntity\": null,\n \"entityType\": \"ASSET\"\n }\n]\n```") @PostMapping("/import") public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); From a1d62373c9784b32c454b45a0fe3b5a409533814 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 26 Apr 2022 12:08:28 +0300 Subject: [PATCH 052/178] Transaction timeout for finding entities by request and importing entities batch --- .../server/service/sync/DefaultEntitiesExportImportService.java | 2 +- .../sync/exporting/DefaultExportableEntitiesService.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 75c4ff9e12..55384b21db 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -70,7 +70,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { fixOrder(exportDataList); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index a616dec6c4..bc8f406c54 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; @@ -130,6 +131,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } + @Transactional(readOnly = true, timeout = 40) @Override public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { switch (request.getType()) { From 335b4ef4652a07d9ddf032941c5c4f7833624ceb Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Sat, 14 May 2022 17:23:34 +0300 Subject: [PATCH 053/178] Improvements for import of rule chains and dashboards --- .../DefaultExportableEntitiesService.java | 3 ++ .../impl/BaseEntityImportService.java | 24 +++++++++++++ .../impl/DashboardImportService.java | 35 +++++------------- .../impl/RuleChainImportService.java | 29 +++++++-------- .../thingsboard/server/utils/RegexUtils.java | 36 +++++++++++++++++++ 5 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/utils/RegexUtils.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index bc8f406c54..ce6924de93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -96,6 +96,9 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } E entity = dao.findById(tenantId, id.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index e96359a746..28095a7b85 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -20,11 +20,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -42,6 +44,7 @@ import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -201,6 +204,27 @@ public abstract class BaseEntityImportService getInternalIdByUuid(UUID externalUuid) { + for (EntityType entityType : EntityType.values()) { + EntityId externalId; + try { + externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); + } catch (Exception e) { + continue; + } + + EntityId internalId = null; + try { + internalId = getInternalId(externalId); + } catch (Exception ignored) {} + + if (internalId != null) { + return Optional.of(internalId); + } + } + return Optional.empty(); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index e7c5ed0ea3..77f32bf94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -27,22 +26,19 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.regex.Pattern; +import java.util.UUID; import java.util.stream.Collectors; @Service @@ -52,7 +48,6 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - Optional.ofNullable(dashboard.getConfiguration()) - .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) - .filter(JsonNode::isObject) - .ifPresent(entityAliases -> entityAliases.forEach(entityAlias -> { - Optional.ofNullable(entityAlias.get("filter")) - .filter(JsonNode::isObject) - .ifPresent(filter -> { - EntityFilter entityFilter = JacksonUtil.treeToValue(filter, EntityFilter.class); - EntityType entityType = DefaultEntityQueryRepository.resolveEntityType(entityFilter); - - String filterJson = filter.toString(); - String newFilterJson = UUID_PATTERN.matcher(filterJson).replaceAll(matchResult -> { - String uuid = matchResult.group(); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.getInternalId(entityId).toString(); - }); - ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); - }); - })); + JsonNode configuration = dashboard.getConfiguration(); + String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + .map(entityId -> entityId.getId().toString()).orElse(uuid); + }); + configuration = JacksonUtil.toJsonNode(newConfigurationJson); + dashboard.setConfiguration(configuration); Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 1e4ed958a8..09a13d5f14 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -16,10 +16,9 @@ package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -29,11 +28,13 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; import java.util.Optional; @@ -67,16 +68,15 @@ public class RuleChainImportService extends BaseEntityImportService { ruleNode.setId(null); ruleNode.setRuleChainId(null); + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); - Optional.ofNullable(ruleNodeConfig) - .flatMap(config -> Optional.ofNullable(config.get("ruleChainId")).filter(JsonNode::isTextual)) - .map(JsonNode::asText).map(UUID::fromString) - .ifPresent(otherRuleChainUuid -> { - ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.getInternalId(new RuleChainId(otherRuleChainUuid)).toString() - )); - ruleNode.setConfiguration(ruleNodeConfig); - }); + String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + .map(entityId -> entityId.getId().toString()) + .orElse(uuid); + }); + ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); + ruleNode.setConfiguration(ruleNodeConfig); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { @@ -84,13 +84,10 @@ public class RuleChainImportService extends BaseEntityImportService replacer) { + return pattern.matcher(s).replaceAll(matchResult -> { + return replacer.apply(matchResult.group()); + }); + } + +} From f82be0153b4e77aba4d686face2b0861ff2af607 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Sat, 2 Apr 2022 19:42:48 +0300 Subject: [PATCH 054/178] Entities VC with Git - initial implementation --- application/pom.xml | 4 + .../EntitiesVersionControlController.java | 118 ++++++++ .../DefaultEntitiesVersionControlService.java | 274 ++++++++++++++++++ .../vcs/EntitiesVersionControlService.java | 50 ++++ .../data/EntitiesVersionControlSettings.java | 28 ++ .../service/sync/vcs/data/EntityVersion.java | 29 ++ .../service/sync/vcs/data/GitSettings.java | 32 ++ .../server/utils/git/Repository.java | 256 ++++++++++++++++ .../server/utils/git/data/Branch.java | 23 ++ .../server/utils/git/data/Commit.java | 25 ++ .../server/utils/git/data/Diff.java | 25 ++ pom.xml | 6 + 12 files changed, 870 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/Repository.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java diff --git a/application/pom.xml b/application/pom.xml index 1b89752925..2bfab2c858 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -337,6 +337,10 @@ Java-WebSocket test + + org.eclipse.jgit + org.eclipse.jgit + diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java new file mode 100644 index 0000000000..10928f1b51 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; + +import java.util.List; +import java.util.Set; +import java.util.UUID; + +@RestController +@RequestMapping("/api/entities/vc") +@RequiredArgsConstructor +public class EntitiesVersionControlController extends BaseController { + + private final DefaultEntitiesVersionControlService versionControlService; + + + + @PostMapping("/version/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); + } + + @GetMapping("/version/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityVersions(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @PathVariable String versionId) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + } + + @PostMapping("/entity/{entityType}/{entityId}/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @PathVariable String versionId) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + } + + + + @GetMapping("/branches") + public Set getAllowedBranches() throws ThingsboardException { + return versionControlService.getAllowedBranches(getTenantId()); + } + + + @PostMapping("/settings") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { + versionControlService.saveSettings(settings); + } + + @GetMapping("/settings") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public EntitiesVersionControlSettings getSettings() { + return versionControlService.getSettings(); + } + + + + @PostMapping("/repository/reset") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public void resetLocalRepository() throws Exception { + versionControlService.resetRepository(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..0d78496f16 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -0,0 +1,274 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import org.thingsboard.server.service.sync.vcs.data.GitSettings; +import org.thingsboard.server.utils.git.Repository; +import org.thingsboard.server.utils.git.data.Commit; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + // TODO [viacheslav]: start up only on one of the cores + + private final TenantService tenantService; + private final EntitiesExportImportService exportImportService; + private final AdminSettingsService adminSettingsService; + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private static final String SETTINGS_KEY = "vc"; + + private Repository repository; + private final ReentrantLock fetchLock = new ReentrantLock(); + private final Lock writeLock = new ReentrantLock(); + + @AfterStartUp + public void init() throws Exception { + try { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null && settings.getGitSettings() != null) { + this.repository = initRepository(settings.getGitSettings()); + } + } catch (Exception e) { + log.error("Failed to initialize entities version control service", e); + } + } + + + @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) + public void fetch() throws Exception { + if (repository == null) return; + + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } + + + @Override + public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { + return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); + } + + @Override + public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportInboundRelations(false) + .exportOutboundRelations(false) + .build(); + List>> entityDataList = entitiesIds.stream() + .map(entityId -> { + return exportImportService.exportEntity(tenantId, entityId, exportSettings); + }) + .collect(Collectors.toList()); + + if (fetchLock.tryLock()) { + try { + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + + writeLock.lock(); + try { + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData> entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), + entityDataJson, StandardCharsets.UTF_8); + } + + Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); + repository.push(); + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } finally { + writeLock.unlock(); + } + } + + + + @Override + public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() + .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) + .collect(Collectors.toList()); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() + .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) + .collect(Collectors.toList()); + } + + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { + checkRepository(); + // FIXME [viacheslav]: validate access + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + @Override + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } + + + + private String getRelativePathForEntity(EntityId entityId) { + return getRelativePathForEntityType(entityId.getEntityType()) + + "/" + entityId.getId() + ".json"; + } + + private String getRelativePathForEntityType(EntityType entityType) { + return entityType.name().toLowerCase(); + } + + + private void checkBranch(TenantId tenantId, String branch) { + if (!getAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to this branch"); + } + } + + public Set getAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptySet()); + } + + @Override + public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { + this.repository = initRepository(settings.getGitSettings()); + + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) + .orElseGet(() -> { + AdminSettings newSettings = new AdminSettings(); + newSettings.setKey(SETTINGS_KEY); + return newSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private static Repository initRepository(GitSettings gitSettings) throws Exception { + if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { + return Repository.open(gitSettings.getRepositoryDirectory(), + gitSettings.getUsername(), gitSettings.getPassword()); + } else { + Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); + return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), + gitSettings.getUsername(), gitSettings.getPassword()); + } + } + + public void resetRepository() throws Exception { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java new file mode 100644 index 0000000000..a9a3c0e3e8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; + + EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; + + + List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + + + void saveSettings(EntitiesVersionControlSettings settings) throws Exception; + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java new file mode 100644 index 0000000000..6a46606845 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.Data; + +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +@Data +public class EntitiesVersionControlSettings { + private Map> allowedBranches; + private GitSettings gitSettings; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java new file mode 100644 index 0000000000..42a2e2f555 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class EntityVersion { + private String id; + private String name; + private String authorName; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java new file mode 100644 index 0000000000..0d97bed2d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class GitSettings { + private String repositoryUri; + private String repositoryDirectory; + private String username; + private String password; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java new file mode 100644 index 0000000000..8099be8259 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -0,0 +1,256 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git; + +import com.google.common.collect.Streams; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.utils.git.data.Commit; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class Repository { + + private final Git git; + private final CredentialsProvider credentialsProvider; + + @Getter + private final String directory; + + private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + this.git = git; + this.credentialsProvider = credentialsProvider; + this.directory = directory; + } + + public static Repository clone(String uri, String directory, + String username, String password) throws GitAPIException { + CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); + Git git = Git.cloneRepository() + .setURI(uri) + .setDirectory(new java.io.File(directory)) + .setNoCheckout(true) + .setCredentialsProvider(credentialsProvider) + .call(); + return new Repository(git, credentialsProvider, directory); + } + + public static Repository open(String directory, String username, String password) throws IOException { + Git git = Git.open(new java.io.File(directory)); + return new Repository(git, newCredentialsProvider(username, password), directory); + } + + + public void fetch() throws GitAPIException { + execute(git.fetch() + .setRemoveDeletedRefs(true)); + } + + + public List listBranches() throws GitAPIException { + return execute(git.branchList() + .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .filter(ref -> !ref.getName().equals(Constants.HEAD)) + .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) + .map(name -> StringUtils.removeStart(name, "origin/")) + .distinct().collect(Collectors.toList()); + } + + + public List listCommits(String branchName, int limit) throws IOException, GitAPIException { + return listCommits(branchName, null, limit); + } + + public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branchName); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + LogCommand command = git.log() + .add(branchId).setMaxCount(limit) + .setRevFilter(RevFilter.NO_MERGES); + if (StringUtils.isNotEmpty(path)) { + command.addPath(path); + } + return Streams.stream(execute(command)) + .map(this::toCommit) + .collect(Collectors.toList()); + } + + + public List listFilesAtCommit(Commit commit) throws IOException { + return listFilesAtCommit(commit, null); + } + + public List listFilesAtCommit(Commit commit, String path) throws IOException { + List files = new ArrayList<>(); + RevCommit revCommit = resolveCommit(commit.getId()); + try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { + treeWalk.reset(revCommit.getTree().getId()); + if (StringUtils.isNotEmpty(path)) { + treeWalk.setFilter(PathFilter.create(path)); + } + treeWalk.setRecursive(true); + while (treeWalk.next()) { + files.add(treeWalk.getPathString()); + } + } + return files; + } + + + public String getFileContentAtCommit(String file, String commitId) throws IOException { + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { + if (treeWalk == null) { + throw new IllegalArgumentException("Not found"); + } + ObjectId blobId = treeWalk.getObjectId(0); + try (ObjectReader objectReader = git.getRepository().newObjectReader()) { + ObjectLoader objectLoader = objectReader.open(blobId); + byte[] bytes = objectLoader.getBytes(); + return new String(bytes, StandardCharsets.UTF_8); + } + } + } + + + public void checkout(String branchName) throws GitAPIException { + execute(git.checkout() + .setName(branchName)); + } + + public void merge(String branchName) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branchName); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + + public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { + execute(git.checkout() + .setOrphan(true) + .setName(name)); + Set uncommittedChanges = git.status().call().getUncommittedChanges(); + if (!uncommittedChanges.isEmpty()) { + RmCommand rm = git.rm(); + uncommittedChanges.forEach(rm::addFilepattern); + execute(rm); + } + execute(git.clean()); + } + + public void clean() throws GitAPIException { + execute(git.clean().setCleanDirectories(true)); + } + + public Commit commit(String message, String filePattern, String author) throws GitAPIException { + execute(git.add().addFilepattern(filePattern)); + RevCommit revCommit = execute(git.commit() + .setMessage(message) + .setAuthor(author, author)); + return toCommit(revCommit); + } + + + public void push() throws GitAPIException { + execute(git.push()); + } + + +// public List getCommitChanges(Commit commit) throws IOException, GitAPIException { +// RevCommit revCommit = resolveCommit(commit.getId()); +// if (revCommit.getParentCount() == 0) { +// return null; // just takes the first parent of a commit, but should find a parent in branch provided +// } +// return execute(git.diff() +// .setOldTree(prepareTreeParser(git.getRepository().parseCommit(revCommit.getParent(0)))) +// .setNewTree(prepareTreeParser(revCommit))).stream() +// .map(diffEntry -> new Diff(diffEntry.getChangeType().name(), diffEntry.getOldPath(), diffEntry.getNewPath())) +// .collect(Collectors.toList()); +// } +// +// +// private AbstractTreeIterator prepareTreeParser(RevCommit revCommit) throws IOException { +// // from the commit we can build the tree which allows us to construct the TreeParser +// //noinspection Duplicates +// org.eclipse.jgit.lib.Repository repository = git.getRepository(); +// try (RevWalk walk = new RevWalk(repository)) { +// RevTree tree = walk.parseTree(revCommit.getTree().getId()); +// +// CanonicalTreeParser treeParser = new CanonicalTreeParser(); +// try (ObjectReader reader = repository.newObjectReader()) { +// treeParser.reset(reader, tree.getId()); +// } +// +// walk.dispose(); +// +// return treeParser; +// } +// } + + private Commit toCommit(RevCommit revCommit) { + return new Commit(revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + } + + private RevCommit resolveCommit(String id) throws IOException { + return git.getRepository().parseCommit(resolve(id)); + } + + private ObjectId resolve(String rev) throws IOException { + return git.getRepository().resolve(rev); + } + + private , T> T execute(C command) throws GitAPIException { + if (command instanceof TransportCommand) { + ((TransportCommand) command).setCredentialsProvider(credentialsProvider); +// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); +// transportCommand.setTransportConfigCallback(transport -> { +// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); +// }); + } + return command.call(); + } + + private static CredentialsProvider newCredentialsProvider(String username, String password) { + return new UsernamePasswordCredentialsProvider(username, password); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java new file mode 100644 index 0000000000..a458f9f5aa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Branch { + private final String shortName; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java new file mode 100644 index 0000000000..7567dba1c5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Commit { + private final String id; + private final String message; + private final String authorName; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java new file mode 100644 index 0000000000..7572a003d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Diff { + private final String type; + private final String oldPath; + private final String newPath; +} diff --git a/pom.xml b/pom.xml index 9713e86993..ea5df86409 100755 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 1.16.0 1.12 + 6.1.0.202203080745-r @@ -1875,6 +1876,11 @@ ${zeroturnaround.version} test + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + From b3dfed5badc997144dc69588aaef2e5ff44b0c79 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 12:00:02 +0300 Subject: [PATCH 055/178] API for listing entities at version, loading all entities at version, listing all available versions --- .../EntitiesVersionControlController.java | 64 +++++- .../DefaultEntitiesVersionControlService.java | 184 +++++++++++++----- .../vcs/EntitiesVersionControlService.java | 11 +- .../server/utils/git/Repository.java | 8 +- 4 files changed, 207 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 10928f1b51..798556998c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities/vc") @@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController { @PostMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, + @PathVariable("entityId") UUID id, @RequestParam String branch, @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); } + @PostMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); + } + + + @GetMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List listEntityVersions(@PathVariable EntityType entityType, @@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); } + @GetMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch) throws Exception { + return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); + } + + @GetMapping("/version") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listVersions(@RequestParam String branch) throws Exception { + return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/files/version/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listFilesAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); + } + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); } @PostMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); + onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + return result; + } + + @PostMapping("/entity/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> loadAllAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + SecurityUser user = getCurrentUser(); + List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); + resultList.forEach(result -> { + onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + }); + return resultList; } @GetMapping("/branches") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") public Set getAllowedBranches() throws ThingsboardException { return versionControlService.getAllowedBranches(getTenantId()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java index 0d78496f16..32da1309fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository; import org.thingsboard.server.utils.git.data.Commit; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -54,7 +57,9 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Service @@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private static final String SETTINGS_KEY = "vc"; private Repository repository; - private final ReentrantLock fetchLock = new ReentrantLock(); - private final Lock writeLock = new ReentrantLock(); + private final Lock fetchLock = new ReentrantLock(); + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); @AfterStartUp public void init() throws Exception { @@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - public void fetch() throws Exception { + private void fetch() throws Exception { if (repository == null) return; - - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); } @@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportOutboundRelations(false) .build(); List>> entityDataList = entitiesIds.stream() - .map(entityId -> { - return exportImportService.exportEntity(tenantId, entityId, exportSettings); - }) + .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) .collect(Collectors.toList()); - if (fetchLock.tryLock()) { - try { - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); - writeLock.lock(); + repositoryLock.writeLock().lock(); try { if (repository.listBranches().contains(branch)) { repository.checkout(branch); @@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); repository.push(); - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + return toVersion(commit); } finally { - writeLock.unlock(); + repositoryLock.writeLock().unlock(); } } @@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); } @Override public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); + return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); + } - return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + @Override + public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { + return listVersions(tenantId, branch, null, limit); + } + + private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, limit).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + + } finally { + repositoryLock.readLock().unlock(); + } } @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - checkRepository(); - // FIXME [viacheslav]: validate access + public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + return repository.listFilesAtCommit(versionId); + } finally { + repositoryLock.readLock().unlock(); + } + } - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return parseEntityData(entityDataJson); + } finally { + repositoryLock.readLock().unlock(); + } } + @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() .importInboundRelations(false) .importOutboundRelations(false) @@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + @Override + public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() + .map(entityDataFilePath -> { + String entityDataJson; + try { + entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return parseEntityData(entityDataJson); + }) + .collect(Collectors.toList()); + + return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + private void tryFetch() throws GitAPIException { + repositoryLock.readLock().lock(); + try { + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } finally { + repositoryLock.readLock().unlock(); + } + } private String getRelativePathForEntity(EntityId entityId) { @@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void checkBranch(TenantId tenantId, String branch) { + // TODO [viacheslav]: all branches are available by default? if (!getAllowedBranches(tenantId).contains(branch)) { throw new IllegalArgumentException("Tenant does not have access to this branch"); } @@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .orElse(Collections.emptySet()); } + private EntityVersion toVersion(Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } + + private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + + @Override public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - this.repository = initRepository(settings.getGitSettings()); + repositoryLock.writeLock().lock(); + try { + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); + } AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) .orElseGet(() -> { @@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } - private void checkRepository() { if (repository == null) { throw new IllegalStateException("Repository is not initialized"); @@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } public void resetRepository() throws Exception { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; + repositoryLock.writeLock().lock(); + try { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java index a9a3c0e3e8..e05ad17826 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -37,10 +37,17 @@ public interface EntitiesVersionControlService { List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; void saveSettings(EntitiesVersionControlSettings settings) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java index 8099be8259..23eb7d7db4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -113,13 +113,13 @@ public class Repository { } - public List listFilesAtCommit(Commit commit) throws IOException { - return listFilesAtCommit(commit, null); + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); } - public List listFilesAtCommit(Commit commit, String path) throws IOException { + public List listFilesAtCommit(String commitId, String path) throws IOException { List files = new ArrayList<>(); - RevCommit revCommit = resolveCommit(commit.getId()); + RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) { From 073875f406ed4437ec844a3a29a116effba5be44 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 28 Apr 2022 13:26:11 +0300 Subject: [PATCH 056/178] Entities VC improvements and refactoring --- .../EntitiesVersionControlController.java | 272 ++++++------ .../DefaultEntitiesVersionControlService.java | 395 ++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 64 +++ .../data/EntitiesVersionControlSettings.java} | 20 +- .../sync/{vcs => vc}/data/EntityVersion.java | 5 +- .../data/EntityVersionLoadResult.java} | 21 +- .../vc/data/EntityVersionLoadSettings.java} | 9 +- .../vc/data/EntityVersionSaveSettings.java} | 6 +- .../sync/vc/data/VersionedEntityInfo.java} | 11 +- .../DefaultEntitiesVersionControlService.java | 358 ---------------- .../vcs/EntitiesVersionControlService.java | 57 --- .../Repository.java => GitRepository.java} | 79 ++-- 12 files changed, 665 insertions(+), 632 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/GitSettings.java => vc/data/EntitiesVersionControlSettings.java} (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs => vc}/data/EntityVersion.java (90%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/EntitiesVersionControlSettings.java => vc/data/EntityVersionLoadResult.java} (57%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Commit.java => service/sync/vc/data/EntityVersionLoadSettings.java} (78%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Branch.java => service/sync/vc/data/EntityVersionSaveSettings.java} (83%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Diff.java => service/sync/vc/data/VersionedEntityInfo.java} (74%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/utils/{git/Repository.java => GitRepository.java} (82%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 798556998c..c39442341e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,159 +16,139 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @RestController @RequestMapping("/api/entities/vc") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { - private final DefaultEntitiesVersionControlService versionControlService; - - - - @PostMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID id, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); - return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); - } - - @PostMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); - return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); - } - - - - @GetMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityVersions(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityTypeVersions(@PathVariable EntityType entityType, - @RequestParam String branch) throws Exception { - return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listVersions(@RequestParam String branch) throws Exception { - return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); - } - - - - @GetMapping("/files/version/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listFilesAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); - } - - - - @GetMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); - } - - @PostMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); - onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - return result; - } - - @PostMapping("/entity/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> loadAllAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - SecurityUser user = getCurrentUser(); - List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); - resultList.forEach(result -> { - onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - }); - return resultList; - } - - - - @GetMapping("/branches") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public Set getAllowedBranches() throws ThingsboardException { - return versionControlService.getAllowedBranches(getTenantId()); - } - - - @PostMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { - versionControlService.saveSettings(settings); - } - - @GetMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public EntitiesVersionControlSettings getSettings() { - return versionControlService.getSettings(); - } - - - - @PostMapping("/repository/reset") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void resetLocalRepository() throws Exception { - versionControlService.resetRepository(); - } + private final EntitiesVersionControlService versionControlService; + + + // search request - export request with settings + +// +// @PostMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID id, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); +// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); +// } +// +// @PostMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, +// @RequestParam UUID[] ids, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// List entitiesIds = Arrays.stream(ids) +// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) +// .collect(Collectors.toList()); +// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); +// } +// +// +// +// @GetMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityVersions(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityTypeVersions(@PathVariable EntityType entityType, +// @RequestParam String branch) throws Exception { +// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listVersions(@RequestParam String branch) throws Exception { +// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); +// } +// +// +// +// @GetMapping("/files/version/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listFilesAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); +// } +// +// +// +// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); +// } +// +// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); +// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// return result; +// } +// +// @PostMapping("/entity/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List>> loadAllAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// SecurityUser user = getCurrentUser(); +// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); +// resultList.forEach(result -> { +// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// }); +// return resultList; +// } +// +// +// +// @GetMapping("/branches") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public Set getAllowedBranches() throws ThingsboardException { +// return versionControlService.getAllowedBranches(getTenantId()); +// } +// +// +// @PostMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { +// versionControlService.saveSettings(settings); +// } +// +// @GetMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public EntitiesVersionControlSettings getSettings() { +// return versionControlService.getSettings(); +// } +// +// +// +// @PostMapping("/repository/reset") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void resetLocalRepository() throws Exception { +// versionControlService.resetRepository(); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..a298d64ccc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,395 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.utils.GitRepository; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final EntitiesExportImportService exportImportService; + + private GitRepository repository; + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); + + private ScheduledExecutorService fetchExecutor; + private ScheduledFuture fetchTask; + + private final AdminSettingsService adminSettingsService; + private static final String SETTINGS_KEY = "vc"; + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + + + @AfterStartUp + public void init() { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null) { + try { + initRepository(settings); + } catch (Exception e) { + log.debug("Failed to init repository", e); + } + } + + int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); + fetchExecutor = Executors.newSingleThreadScheduledExecutor(); + fetchTask = scheduleFetch(fetchPeriod); + } + + + @Override + public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); + } + + @Override + public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + checkRepository(); + checkBranch(user.getTenantId(), branch); + + List> entityDataList = new ArrayList<>(); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(settings.isSaveRelations()) + .build(); + for (EntityId entityId : entitiesIds) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); + } + + GitRepository.Commit commit = repository.commit(versionName, "."); + repository.push(); + return toVersion(commit); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @Override + public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch) throws Exception { + return listVersions(tenantId, branch, null); + } + + private List listVersions(TenantId tenantId, String branch, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + } + + @Override + public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + checkVersion(tenantId, branch, versionId, path); + + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + info.setEntityName(entityData.getEntity().getName()); + return info; + }) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); + } + + @Override + public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); + } + + @Override + public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, null, settings); + } + + private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { + List> entityDataList = new ArrayList<>(); + repositoryLock.readLock().lock(); + try { + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { + EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + entityDataList.add(entityData); + } + } finally { + repositoryLock.readLock().unlock(); + } + + EntityImportSettings importSettings = EntityImportSettings.builder() + .updateRelations(settings.isLoadRelations()) + .findExistingByName(settings.isFindExistingEntityByName()) + .build(); + List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); + + return importResults.stream() + .map(importResult -> EntityVersionLoadResult.builder() + .previousEntityVersion(importResult.getOldEntity()) + .newEntityVersion(importResult.getSavedEntity()) + .entityType(importResult.getEntityType()) + .build()) + .collect(Collectors.toList()); + } + + @SneakyThrows + private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { + repositoryLock.readLock().lock(); + try { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + private void fetch() throws GitAPIException { + repositoryLock.writeLock().lock(); + try { + repository.fetch(); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private ScheduledFuture scheduleFetch(int fetchPeriod) { + return fetchExecutor.scheduleWithFixedDelay(() -> { + if (repository == null) return; + try { + fetch(); + } catch (Exception e) { + log.error("Failed to fetch remote repository", e); + } + }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + } + + + private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Version not found"); + } + } + + @Override + public List listAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptyList()); + } + + private void checkBranch(TenantId tenantId, String branch) { + if (!listAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to the branch"); + } + } + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private void initRepository(EntitiesVersionControlSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { + this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(Path.of(settings.getRepositoryDirectory())); + this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), + settings.getUsername(), settings.getPassword()); + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private void clearRepository() throws IOException { + repositoryLock.writeLock().lock(); + try { + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repository = null; + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @SneakyThrows + @Override + public void saveSettings(EntitiesVersionControlSettings settings) { + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .orElseGet(() -> { + AdminSettings newAdminSettings = new AdminSettings(); + newAdminSettings.setKey(SETTINGS_KEY); + return newAdminSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + + repositoryLock.writeLock().lock(); + try { + clearRepository(); + initRepository(settings); + } finally { + repositoryLock.writeLock().unlock(); + } + + if (settings.getFetchPeriod() != 0) { + fetchTask.cancel(true); + fetchTask = scheduleFetch(settings.getFetchPeriod()); + } + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..435c661b02 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + + List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + + List listVersions(TenantId tenantId, String branch) throws Exception; + + + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + + List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + + List listAllowedBranches(TenantId tenantId); + + + void saveSettings(EntitiesVersionControlSettings settings); + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 0d97bed2d1..06e0a9c7aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -13,20 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; +import java.util.UUID; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class GitSettings { - private String repositoryUri; +public class EntitiesVersionControlSettings { private String repositoryDirectory; + private String repositoryUri; private String username; private String password; + + private int fetchPeriod; + + private Map> tenantsAllowedBranches; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java index 42a2e2f555..3547f89e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@NoArgsConstructor @AllArgsConstructor +@NoArgsConstructor public class EntityVersion { private String id; private String name; - private String authorName; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java index 6a46606845..4cf0ee2b0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java @@ -13,16 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; - -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; @Data -public class EntitiesVersionControlSettings { - private Map> allowedBranches; - private GitSettings gitSettings; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionLoadResult { + private ExportableEntity newEntityVersion; + private ExportableEntity previousEntityVersion; + private EntityType entityType; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java index 7567dba1c5..842f6ddd4e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Commit { - private final String id; - private final String message; - private final String authorName; +public class EntityVersionLoadSettings { + private boolean loadRelations; + private boolean findExistingEntityByName; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java index a458f9f5aa..1fc36463ba 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Branch { - private final String shortName; +public class EntityVersionSaveSettings { + private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 7572a003d2..97bb6ada12 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; @Data -public class Diff { - private final String type; - private final String oldPath; - private final String newPath; +public class VersionedEntityInfo { + private EntityId externalId; + private String entityName; + // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java deleted file mode 100644 index 32da1309fa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; -import org.thingsboard.server.service.sync.vcs.data.GitSettings; -import org.thingsboard.server.utils.git.Repository; -import org.thingsboard.server.utils.git.data.Commit; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - // TODO [viacheslav]: start up only on one of the cores - - private final TenantService tenantService; - private final EntitiesExportImportService exportImportService; - private final AdminSettingsService adminSettingsService; - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private static final String SETTINGS_KEY = "vc"; - - private Repository repository; - private final Lock fetchLock = new ReentrantLock(); - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - @AfterStartUp - public void init() throws Exception { - try { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null && settings.getGitSettings() != null) { - this.repository = initRepository(settings.getGitSettings()); - } - } catch (Exception e) { - log.error("Failed to initialize entities version control service", e); - } - } - - - @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - private void fetch() throws Exception { - if (repository == null) return; - tryFetch(); - } - - - @Override - public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { - return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); - } - - @Override - public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(false) - .exportOutboundRelations(false) - .build(); - List>> entityDataList = entitiesIds.stream() - .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) - .collect(Collectors.toList()); - - tryFetch(); - - repositoryLock.writeLock().lock(); - try { - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); - } - - for (EntityExportData> entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), - entityDataJson, StandardCharsets.UTF_8); - } - - Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); - repository.push(); - return toVersion(commit); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - - - @Override - public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); - } - - @Override - public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); - } - - @Override - public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, null, limit); - } - - private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { - repositoryLock.readLock().lock(); - try { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, path, limit).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - return repository.listFilesAtCommit(versionId); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return parseEntityData(entityDataJson); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); - return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } - - @Override - public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() - .map(entityDataFilePath -> { - String entityDataJson; - try { - entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return parseEntityData(entityDataJson); - }) - .collect(Collectors.toList()); - - return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } finally { - repositoryLock.readLock().unlock(); - } - } - - private void tryFetch() throws GitAPIException { - repositoryLock.readLock().lock(); - try { - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } - } finally { - repositoryLock.readLock().unlock(); - } - } - - - private String getRelativePathForEntity(EntityId entityId) { - return getRelativePathForEntityType(entityId.getEntityType()) - + "/" + entityId.getId() + ".json"; - } - - private String getRelativePathForEntityType(EntityType entityType) { - return entityType.name().toLowerCase(); - } - - - private void checkBranch(TenantId tenantId, String branch) { - // TODO [viacheslav]: all branches are available by default? - if (!getAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to this branch"); - } - } - - public Set getAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptySet()); - } - - private EntityVersion toVersion(Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); - } - - private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); - } - - - - @Override - public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .orElseGet(() -> { - AdminSettings newSettings = new AdminSettings(); - newSettings.setKey(SETTINGS_KEY); - return newSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - } - - @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) - .orElse(null); - } - - - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } - } - - private static Repository initRepository(GitSettings gitSettings) throws Exception { - if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { - return Repository.open(gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } else { - Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); - return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } - } - - public void resetRepository() throws Exception { - repositoryLock.writeLock().lock(); - try { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; - } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java deleted file mode 100644 index e05ad17826..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.List; - -public interface EntitiesVersionControlService { - - EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; - - EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; - - - List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; - - List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; - - List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - - - List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - void saveSettings(EntitiesVersionControlSettings settings) throws Exception; - - EntitiesVersionControlSettings getSettings(); - -} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/utils/git/Repository.java rename to application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 23eb7d7db4..205bbca50e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git; +package org.thingsboard.server.utils; import com.google.common.collect.Streams; +import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.Git; @@ -32,19 +33,21 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.utils.git.data.Commit; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class Repository { +public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; @@ -52,13 +55,13 @@ public class Repository { @Getter private final String directory; - private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; this.directory = directory; } - public static Repository clone(String uri, String directory, + public static GitRepository clone(String uri, String directory, String username, String password) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() @@ -67,12 +70,12 @@ public class Repository { .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new Repository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory); } - public static Repository open(String directory, String username, String password) throws IOException { + public static GitRepository open(String directory, String username, String password) throws IOException { Git git = Git.open(new java.io.File(directory)); - return new Repository(git, newCredentialsProvider(username, password), directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory); } @@ -81,6 +84,20 @@ public class Repository { .setRemoveDeletedRefs(true)); } + public void checkout(String branch) throws GitAPIException { + execute(git.checkout() + .setName(branch)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + public List listBranches() throws GitAPIException { return execute(git.branchList() @@ -92,12 +109,12 @@ public class Repository { } - public List listCommits(String branchName, int limit) throws IOException, GitAPIException { - return listCommits(branchName, null, limit); + public List listCommits(String branch, int limit) throws IOException, GitAPIException { + return listCommits(branch, null, limit); } - public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); + public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } @@ -150,20 +167,6 @@ public class Repository { } - public void checkout(String branchName) throws GitAPIException { - execute(git.checkout() - .setName(branchName)); - } - - public void merge(String branchName) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); - if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); - } - execute(git.merge() - .include(branchId)); - } - public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) @@ -177,15 +180,11 @@ public class Repository { execute(git.clean()); } - public void clean() throws GitAPIException { - execute(git.clean().setCleanDirectories(true)); - } - public Commit commit(String message, String filePattern, String author) throws GitAPIException { - execute(git.add().addFilepattern(filePattern)); + public Commit commit(String message, String filesPattern) throws GitAPIException { + execute(git.add().addFilepattern(filesPattern)); RevCommit revCommit = execute(git.commit() - .setMessage(message) - .setAuthor(author, author)); + .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); } @@ -239,12 +238,8 @@ public class Repository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand) { + if (command instanceof TransportCommand && credentialsProvider != null) { ((TransportCommand) command).setCredentialsProvider(credentialsProvider); -// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); -// transportCommand.setTransportConfigCallback(transport -> { -// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); -// }); } return command.call(); } @@ -253,4 +248,12 @@ public class Repository { return new UsernamePasswordCredentialsProvider(username, password); } + + @Data + public static class Commit { + private final String id; + private final String message; + private final String authorName; + } + } From 074ea8af6cdd9d723f57dd55615cf602d16738a8 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 16:09:00 +0300 Subject: [PATCH 057/178] VC refactoring --- .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesExportImportController.java | 206 -------- .../EntitiesVersionControlController.java | 293 ++++++----- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 30 +- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 56 ++- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 12 +- .../exporting/data/DeviceExportData.java | 2 +- .../exporting/data/EntityExportData.java | 2 +- .../exporting/data}/EntityExportSettings.java | 2 +- .../exporting/data/RuleChainExportData.java | 2 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 9 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/data/EntityImportResult.java | 2 +- .../importing/data/EntityImportSettings.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../importing/data/request/ImportRequest.java | 32 -- .../DefaultEntitiesVersionControlService.java | 463 +++++++++++------- .../vc/EntitiesVersionControlService.java | 27 +- .../data/EntitiesVersionControlSettings.java | 10 +- .../sync/vc/data/VersionCreationResult.java | 11 + ...LoadResult.java => VersionLoadResult.java} | 8 +- ...iesByCustomFilterVersionCreateConfig.java} | 14 +- ...tiesByCustomQueryVersionCreateConfig.java} | 12 +- .../EntityListVersionCreateConfig.java} | 10 +- .../EntityTypeVersionCreateConfig.java} | 14 +- .../SingleEntityVersionCreateConfig.java} | 10 +- .../request/create/VersionCreateConfig.java} | 18 +- .../create/VersionCreateConfigType.java} | 4 +- .../create/VersionCreateRequest.java} | 14 +- .../load/EntityTypeVersionLoadConfig.java | 12 + .../load/EntityTypeVersionLoadRequest.java | 20 + .../request/load/EntityVersionLoadConfig.java | 11 + .../load/SingleEntityVersionLoadRequest.java | 20 + .../load/VersionLoadRequest.java} | 11 +- .../request/load/VersionLoadRequestType.java | 6 + .../server/utils/GitRepository.java | 39 +- ...aseEntitiesExportImportControllerTest.java | 20 +- ...EntitiesExportImportControllerSqlTest.java | 53 +- .../org/thingsboard/server/dao/DaoUtil.java | 20 +- .../server/dao/ExportableEntityDao.java | 4 + .../server/dao/sql/asset/JpaAssetDao.java | 5 + .../dao/sql/customer/JpaCustomerDao.java | 5 + .../sql/dashboard/DashboardRepository.java | 4 + .../dao/sql/dashboard/JpaDashboardDao.java | 7 + .../server/dao/sql/device/JpaDeviceDao.java | 5 + .../dao/sql/device/JpaDeviceProfileDao.java | 5 + .../server/dao/sql/rule/JpaRuleChainDao.java | 5 + 68 files changed, 860 insertions(+), 777 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/DefaultEntitiesExportImportService.java (84%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/EntitiesExportImportService.java (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/DefaultExportableEntitiesService.java (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/ExportableEntitiesService.java (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/DeviceExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/EntityExportData.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request => exportimport/exporting/data}/EntityExportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/RuleChainExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/AbstractBulkImportService.java (97%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportResult.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/RuleChainImportService.java (95%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadResult.java => VersionLoadResult.java} (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityFilterExportRequest.java => vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java} (69%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityQueryExportRequest.java => vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java} (72%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityListExportRequest.java => vc/data/request/create/EntityListVersionCreateConfig.java} (73%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityTypeExportRequest.java => vc/data/request/create/EntityTypeVersionCreateConfig.java} (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/SingleEntityExportRequest.java => vc/data/request/create/SingleEntityVersionCreateConfig.java} (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequest.java => vc/data/request/create/VersionCreateConfig.java} (58%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequestType.java => vc/data/request/create/VersionCreateConfigType.java} (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadSettings.java => request/create/VersionCreateRequest.java} (73%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionSaveSettings.java => request/load/VersionLoadRequest.java} (74%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index e48f2dcf23..66b570cf1d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -54,8 +54,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 03f0996ea9..4b6eca7879 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,8 +74,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8f45427603..948a380fcd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -55,8 +55,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java deleted file mode 100644 index c91fd0c292..0000000000 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import io.swagger.annotations.ApiOperation; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; - -@RestController -@RequestMapping("/api/entities") -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class EntitiesExportImportController extends BaseController { - - private final EntitiesExportImportService exportImportService; - private final ExportableEntitiesService exportableEntitiesService; - - - @ApiOperation(value = "Export entities by request", notes = "" + - "Takes export request and returns list of export data for each entity found by request. " + - "Supported entity types for export, hence for import, are **DEVICE**, **DEVICE_PROFILE**, **ASSET**, " + - "**CUSTOMER**, **RULE_CHAIN** and **DASHBOARD**." + NEW_LINE + - "For each type of export request, you can set some export settings: \n" + - "- **exportRelations** - whether to export inbound and outbound relations for an entity " + - "(only relations of type group COMMON can be exported)" + NEW_LINE + - "Supported export requests:\n" + - "- **SINGLE_ENTITY**:" + NEW_LINE + - " To export a single entity by id. Example:" + NEW_LINE + - "```\n{\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **ENTITY_LIST**:" + NEW_LINE + - " To export a list of entities by their ids. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"ASSET\",\n \"id\": \"2f0a3bd0-989d-11ec-93b5-6de6c2b68078\"\n }\n ],\n \"exportSettings\": {\n \"exportRelations\": true\n }\n}\n```" + NEW_LINE + - "- **ENTITY_TYPE**:" + NEW_LINE + - " To export entities of specified entity type. You need to specify page size, " + - "and may specify page index and customer id (to limit the list of entities to the ones owned by a customer). " + - "Entities are ordered by created time descendingly. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_FILTER**:" + NEW_LINE + - " To export entities by custom entity filter. The order used is the same as for ENTITY_TYPE export request. Example:" + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"Thermostats\",\n \"deviceNameFilter\": \"\"\n },\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_QUERY**:" + NEW_LINE + - " To export entities by custom entity query. Example: " + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_QUERY\",\n \"query\": {\n \"entityFilter\": {\n \"type\": \"entityType\",\n \"entityType\": \"DEVICE\"\n },\n \"pageLink\": {\n \"page\": 0,\n \"pageSize\": 200,\n \"textSearch\": \"THB_\",\n \"sortOrder\": {\n \"key\": {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n },\n \"direction\": \"DESC\"\n }\n },\n \"entityFields\": [\n {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n }\n ],\n \"latestValues\": [\n {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n }\n ],\n \"keyFilters\": [\n {\n \"key\": {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n },\n \"valueType\": \"NUMERIC\",\n \"predicate\": {\n \"type\": \"NUMERIC\",\n \"operation\": \"GREATER\",\n \"value\": {\n \"defaultValue\": 0\n }\n }\n }\n ]\n },\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "Mostly, export data of an entity contains the whole entity itself and its relations " + - "(if option to export relations was enabled):" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"ASSET\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1648204424029,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": { ... },\n \"customerId\": { ... },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n ...\n },\n \"relations\": [\n {\n \"from\": {\n \"entityType\": \"ASSET\",\n \"id\": ...\n },\n \"to\": {\n \"entityType\": \"DEVICE\",\n \"id\": ...\n },\n \"type\": \"Contains\",\n \"typeGroup\": \"COMMON\",\n \"additionalInfo\": {\n \"a\": \"b\"\n }\n }\n ]\n }\n]\n```" + NEW_LINE + - "For devices, export data will additionally contain device's credentials; for rule chains - its metadata:" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"DEVICE\",\n \"entity\": { ... },\n \"credentials\": {\n \"id\": { ... },\n \"createdTime\": 1648829321209,\n \"deviceId\": { ... },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"5cZEDo45KGW7JgVNv4Ko\",\n \"credentialsValue\": null\n }\n }\n]\n```" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"RULE_CHAIN\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1646056614257,\n \"additionalInfo\": null,\n \"tenantId\": { ... },\n \"name\": \"Rule Chain 2\",\n \"type\": \"CORE\",\n \"firstRuleNodeId\": { ... },\n \"root\": false,\n ...\n },\n \"metaData\": {\n \"ruleChainId\": { ... },\n \"firstNodeIndex\": 7,\n \"nodes\": [ ... ],\n \"connections\": [ ... ],\n \"ruleChainConnections\": null\n }\n }\n]\n```" + NEW_LINE + - "Returned export data is to be used later for import request." + NEW_LINE + - "If any entity found by request is of unsupported type - an error will be returned.\n" + - "Also, if a user does not have a READ permission for an entity (or, if relations are exported, for a bounded entity), " + - "access will be denied." + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping("/export") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - return exportEntitiesByRequest(user, exportRequest); - } catch (Exception e) { - log.warn("Failed to export entities for request {}", exportRequest, e); - throw handleException(e); - } - } - - @ApiOperation(value = "Export entities by multiple requests", notes = "" + - "The API behaviour is the same as for exporting entities by single request, " + - "except that this method takes an array of export requests as a request body." + NEW_LINE + - "Example:" + NEW_LINE + - "```\n[\n {\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"5f9eda10-b442-11ec-bbf5-adec34031568\"\n }\n },\n {\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"thermostat\",\n \"deviceNameFilter\": \"\"\n },\n \"pageSize\": 1000\n },\n {\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"pageSize\": 1000,\n \"exportSettings\": {\n \"exportRelations\": true\n }\n },\n {\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"2ef13590-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"e7311ec0-b442-11ec-bbf5-adec34031568\"\n }\n ]\n }\n]\n```" + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping(value = "/export", params = {"multiple"}) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - List> result = new ArrayList<>(); - for (ExportRequest exportRequest : exportRequests) { - List> exportDataList = exportEntitiesByRequest(user, exportRequest); - result.addAll(exportDataList); - } - return result; - } catch (Exception e) { - log.warn("Failed to export entities for requests {}", exportRequests, e); - throw handleException(e); - } - } - - private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { - List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); - - EntityExportSettings exportSettings = exportRequest.getExportSettings(); - if (exportSettings == null) { - exportSettings = EntityExportSettings.builder() - .exportRelations(false) - .build(); - } - - List> exportDataList = new ArrayList<>(); - for (EntityId entityId : entities) { - EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportSettings); - exportDataList.add(exportData); - } - return exportDataList; - } - - - @ApiOperation(value = "Import entities by request", notes = "" + - "Takes import request and returns the list of import results. " + - "Import request must contain the list of export data and might contain import settings. " + NEW_LINE + - "The method creates an entity if it is new in the scope of a tenant, or otherwise updates an existing one. " + - "On entity import request, we first try to find an entity within a tenant that has externalId equal " + - "to the id in the export data. If the platform fails to do that, we then search for an entity with " + - "regular (internal) id like the one in the export data (this is useful in case we are exporting and " + - "importing entities within the same tenant). Then, if we still haven't found any entity, if findExistingByName " + - "option of the EntityImportSettings is enabled, we will search for the one by its name (this is also useful " + - "for avoiding conflicts with default device profile or Root Rule Chain when importing all entities from another " + - "tenant). After, if the exported entity is new for this tenant, we simply save it with external id " + - "from the export data, and also create relations if any (if updateRelations option is enabled). " + - "Otherwise, we will reset all fields of the existing entity to the ones from the export data and save it, " + - "and also will update the list of relations (remove the ones that aren't present in the export data, " + - "and update or create others), if updateRelations option is enabled." + NEW_LINE + - "If an entity contains references to some other entities, like device references certain device profile, " + - "we will find this other entity within the tenant by this principle: look for an entity with " + - "such external id, or otherwise, internal id. This requires referenced entities to be imported " + - "before the referencing entity (if we are importing to another tenant). So, when receiving the list " + - "of entities' export data for import, we first try to fix the data order to import 'standalone' entities first." + NEW_LINE + - "As for relations importing, they are processed after all entities in the import batch are already saved, " + - "and the internal id of a bounded entity is found with the regular principle." + NEW_LINE + - "Import of all entities and their relations from the import request is processed in the single transaction, " + - "and so everything will be rolled back if the platform fails to e.g. find internal entity by external id. \n" + - "Example of import request:\n" + - "```\n{\n \"importSettings\": {\n \"findExistingByName\": false,\n \"updateRelations\": false\n },\n \"exportDataList\": [\n {\n \"entityType\": \"DEVICE_PROFILE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n \"createdTime\": 1649096026765,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"name\": \"Profile 1\",\n ...\n }\n },\n {\n \"entityType\": \"DEVICE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154276962,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"customerId\": {\n \"entityType\": \"CUSTOMER\",\n \"id\": \"13814000-1dd2-11b2-8080-808080808080\"\n },\n \"name\": \"Device 1\",\n \"type\": \"Profile 1\",\n \"label\": \"v1.0\",\n \"deviceProfileId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n ...\n },\n \"credentials\": {\n \"id\": {\n \"id\": \"981e0360-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154277014,\n \"deviceId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"sGExNdnl71uKmkNvtNdp\",\n \"credentialsValue\": null\n }\n }\n ]\n}\n```" + NEW_LINE + - "The response contains a list of EntityImportResult which has values of savedEntity and oldEntity:\n" + - "```\n[\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v2.0\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"6b03ab20-989e-11ec-b446-89df822d7fa2\"\n }\n },\n \"oldEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v1.0\",\n ...\n \"externalId\": null\n },\n \"entityType\": \"ASSET\"\n },\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"387213f0-b4ea-11ec-abfc-6dcc6508d0b5\"\n },\n \"createdTime\": 1649167860399,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 2\",\n \"type\": \"B\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"0b9ea0d0-ac27-11ec-a2a6-89d15eae3b21\"\n }\n },\n \"oldEntity\": null,\n \"entityType\": \"ASSET\"\n }\n]\n```") - @PostMapping("/import") - public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - EntityImportSettings importSettings = importRequest.getImportSettings(); - if (importSettings == null) { - importSettings = EntityImportSettings.builder() - .findExistingByName(false) - .updateRelations(false) - .build(); - } - - List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importSettings); - - importResults.stream() - .map(EntityImportResult::getSendEventsCallback) - .filter(Objects::nonNull) - .forEach(sendEventsCallback -> { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send event for entity", e); - } - }); - - return importResults; - } catch (Exception e) { - log.warn("Failed to import entities for request {}", importRequest, e); - throw handleException(e); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index c39442341e..0625dbeb38 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,140 +15,189 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; @RestController @RequestMapping("/api/entities/vc") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; - // search request - export request with settings - -// -// @PostMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID id, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); -// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); -// } -// -// @PostMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, -// @RequestParam UUID[] ids, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// List entitiesIds = Arrays.stream(ids) -// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) -// .collect(Collectors.toList()); -// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); -// } -// -// -// -// @GetMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityVersions(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityTypeVersions(@PathVariable EntityType entityType, -// @RequestParam String branch) throws Exception { -// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listVersions(@RequestParam String branch) throws Exception { -// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); -// } -// -// -// -// @GetMapping("/files/version/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listFilesAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); -// } -// -// -// -// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); -// } -// -// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); -// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// return result; -// } -// -// @PostMapping("/entity/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List>> loadAllAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// SecurityUser user = getCurrentUser(); -// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); -// resultList.forEach(result -> { -// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// }); -// return resultList; -// } -// -// -// -// @GetMapping("/branches") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public Set getAllowedBranches() throws ThingsboardException { -// return versionControlService.getAllowedBranches(getTenantId()); -// } -// -// -// @PostMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { -// versionControlService.saveSettings(settings); -// } -// -// @GetMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public EntitiesVersionControlSettings getSettings() { -// return versionControlService.getSettings(); -// } -// -// -// -// @PostMapping("/repository/reset") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void resetLocalRepository() throws Exception { -// versionControlService.resetRepository(); -// } + @PostMapping("/version") + public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + return versionControlService.saveEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") + public List listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { + try { + EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}/{entityType}") + public List listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { + try { + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}") + public List listVersions(@PathVariable String branch) throws ThingsboardException { + try { + return versionControlService.listVersions(getTenantId(), branch); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/entity/{branch}/{entityType}/{versionId}") + public List listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/entity/{branch}/{versionId}") + public List listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + + @PostMapping("/entity") + public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + String versionId = request.getVersionId(); + if (versionId == null) { + List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); + if (versions.size() > 0) { + versionId = versions.get(0).getId(); + } else { + throw new IllegalArgumentException("No versions available in branch"); + } + } + + return versionControlService.loadEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "") + @GetMapping("/branches") + public List listBranches() throws ThingsboardException { + try { + List remoteBranches = versionControlService.listBranches(getTenantId()); + List infos = new ArrayList<>(); + + String defaultBranch = getSettings().getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } + + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @GetMapping("/settings") + public EntitiesVersionControlSettings getSettings() throws ThingsboardException { + try { + return versionControlService.getSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @PostMapping("/settings") + public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + versionControlService.saveSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + + @Data + public static class BranchInfo { + private final String name; + private final boolean isDefault; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index cfefcafbf4..179c9c697d 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 3aac9c3641..3e63652b97 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index b09ac532df..75f7133721 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 55384b21db..608a256514 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,14 +27,14 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; import java.util.ArrayList; @@ -74,6 +74,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { fixOrder(exportDataList); + List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { @@ -88,6 +89,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS saveReferencesCallback.run(); } + importResults.stream() + .map(EntityImportResult::getSendEventsCallback) + .filter(Objects::nonNull) + .forEach(sendEventsCallback -> { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + return importResults; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index eaaf01ba2c..5c29feb57c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index ce6924de93..4d772ad935 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -37,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.entity.EntityService; @@ -46,12 +47,12 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -129,6 +130,16 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi return entity; } + @Override + public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + return exportableEntityDao.findByTenantId(tenantId.getId(), pageLink); + } + return new PageData<>(); + } + private boolean belongsToTenant(HasId entity, TenantId tenantId) { return tenantId.equals(((HasTenantId) entity).getTenantId()); } @@ -136,31 +147,31 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi @Transactional(readOnly = true, timeout = 40) @Override - public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { + public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { switch (request.getType()) { case SINGLE_ENTITY: { - return List.of(((SingleEntityExportRequest) request).getEntityId()); + return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); } case ENTITY_LIST: { - return ((EntityListExportRequest) request).getEntitiesIds(); + return ((EntityListVersionCreateConfig) request).getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); entityTypeFilter.setEntityType(exportRequest.getEntityType()); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_FILTER: { - CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; + EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; EntityFilter filter = exportRequest.getFilter(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_QUERY: { - CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; + EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; EntityDataQuery query = exportRequest.getQuery(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); @@ -194,6 +205,17 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } } + @Override + public void deleteByTenantIdAndId(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + + dao.removeById(tenantId, id.getId()); + } + @Override public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java index 73f946e722..5532f30125 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 9021a03e85..043ec91e20 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,9 +21,11 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import java.util.List; @@ -35,7 +37,11 @@ public interface ExportableEntitiesService { , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); - List findEntitiesForRequest(TenantId tenantId, ExportRequest request); + , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + + List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: + + void deleteByTenantIdAndId(TenantId tenantId, I id); void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java index 56633f9c91..4be338d7cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java index 728d8212c4..81e3b344d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java index 5454105ee5..0fb318b5fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java index 2e05e2f1f5..e1d870473f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java index 7b5f758f93..46428a763f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java index a051f0e59f..69af613480 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java index 21df8beda2..93f9c51655 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java index e814cd6ce7..f40ee6884a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java index 6c82530273..be6f62efe4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.exportimport.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java index 06cd7a93be..b2bfafc695 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -166,7 +165,7 @@ public abstract class AbstractBulkImportService data) { + private void saveKvs(SecurityUser user, E entity, Map data) { Arrays.stream(BulkImportColumnType.values()) .filter(BulkImportColumnType::isKv) .map(kvType -> { @@ -250,7 +249,7 @@ public abstract class AbstractBulkImportService columnsMappings = request.getMapping().getColumns(); + List columnsMappings = request.getMapping().getColumns(); return records.stream() .map(record -> { EntityData entityData = new EntityData(); @@ -281,7 +280,7 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); - private final Map kvs = new LinkedHashMap<>(); + private final Map kvs = new LinkedHashMap<>(); private int lineNumber; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java index beafe5e95f..9c0e6b6491 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java index f600289ab6..6347910cab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java index da12a6baeb..550d4a72f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java index 68e7a49b18..a39dcc7481 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java index 62c574d5f0..242856bfcb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java index a5de7d8744..075a6896ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java index ae54a988f3..80a942c73f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java index 28095a7b85..7ecfa7d3ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java index 1a79fe666b..b69c5a0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java index 77f32bf94c..b85e403f87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java index 091b6eb856..112152365b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java index 5bc45dc567..f97950aaf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java index 09a13d5f14..993f9fa27a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java deleted file mode 100644 index 7943e1a404..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.importing.data.request; - -import lombok.Data; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; - -import java.util.List; - -@Data -public class ImportRequest { - - private List> exportDataList; - private EntityImportSettings importSettings; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index a298d64ccc..919960ce62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,35 +18,62 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import org.thingsboard.server.utils.GitRepository; import java.io.File; @@ -56,16 +83,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -73,75 +103,152 @@ import java.util.stream.Collectors; public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { private final EntitiesExportImportService exportImportService; + private final ExportableEntitiesService exportableEntitiesService; + private final AttributesService attributesService; + private final EntityService entityService; + private final TenantDao tenantDao; - private GitRepository repository; - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - private ScheduledExecutorService fetchExecutor; - private ScheduledFuture fetchTask; + // TODO [viacheslav]: concurrency + private final Map repositories = new ConcurrentHashMap<>(); + @Value("${java.io.tmpdir}") + private String repositoriesFolder; - private final AdminSettingsService adminSettingsService; private static final String SETTINGS_KEY = "vc"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null) { - try { - initRepository(settings); - } catch (Exception e) { - log.debug("Failed to init repository", e); + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } } - } - - int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); - fetchExecutor = Executors.newSingleThreadScheduledExecutor(); - fetchTask = scheduleFetch(fetchPeriod); + }); + Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, 5, 5, TimeUnit.SECONDS); } @Override - public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); - } + public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); + repository.getLock().writeLock().lock(); - @Override - public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - repositoryLock.writeLock().lock(); try { - checkRepository(); - checkBranch(user.getTenantId(), branch); - - List> entityDataList = new ArrayList<>(); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(settings.isSaveRelations()) - .build(); - for (EntityId entityId : entitiesIds) { - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // FIXME [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); + for (VersionCreateRequest.Config config : request.getConfigs()) { + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build(); + + List> entityDataList = new ArrayList<>(); + for (EntityId entityId : findEntities()) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + if (config.isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() + .map(EntityExportData::getEntityType) + .distinct() + .forEach(entityType -> { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + } } + // TODO [viacheslav]: find with pagination - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); - } + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - GitRepository.Commit commit = repository.commit(versionName, "."); + GitRepository.Commit commit = repository.commit(request.getVersionName()); repository.push(); - return toVersion(commit); + + result.setVersion(toVersion(commit)); + return result; } finally { - repositoryLock.writeLock().unlock(); + repository.getLock().writeLock().unlock(); + } + } + + private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { + switch (entityFilter.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + return List.of(filter.getEntityId()); + } + case ENTITY_LIST: { + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + return filter.getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); + entityTypeFilter.setEntityType(filter.getEntityType()); + + newFilter.setFilter(entityTypeFilter); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_FILTER: { + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); + + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + newFilter.setQuery(query); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_QUERY: { + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); + return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() + .stream().map(EntityData::getEntityId) + .collect(Collectors.toList()); + } } } @@ -162,16 +269,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() .map(this::toVersion) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @@ -187,16 +292,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - checkVersion(tenantId, branch, versionId, path); - + checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); VersionedEntityInfo info = new VersionedEntityInfo(); info.setExternalId(entityId); @@ -205,171 +308,155 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @Override - public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); - } - - @Override - public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); - } - - @Override - public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, null, settings); - } + public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); - private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { List> entityDataList = new ArrayList<>(); - repositoryLock.readLock().lock(); + EntityVersion version; + repository.getLock().readLock().lock(); try { - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { - EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { + EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); entityDataList.add(entityData); } } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } EntityImportSettings importSettings = EntityImportSettings.builder() .updateRelations(settings.isLoadRelations()) .findExistingByName(settings.isFindExistingEntityByName()) .build(); + // FIXME [viacheslav]: do evrth in transaction List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - return importResults.stream() - .map(importResult -> EntityVersionLoadResult.builder() - .previousEntityVersion(importResult.getOldEntity()) - .newEntityVersion(importResult.getSavedEntity()) - .entityType(importResult.getEntityType()) - .build()) - .collect(Collectors.toList()); + Map results = new HashMap<>(); + + boolean removeNonexistentLocalEntities = false; + if () + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); + } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); + }); + } + + return new ArrayList<>(results.values()); } @SneakyThrows - private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { - repositoryLock.readLock().lock(); + private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); return JacksonUtil.fromString(entityDataJson, EntityExportData.class); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } - - private void fetch() throws GitAPIException { - repositoryLock.writeLock().lock(); - try { - repository.fetch(); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - private ScheduledFuture scheduleFetch(int fetchPeriod) { - return fetchExecutor.scheduleWithFixedDelay(() -> { - if (repository == null) return; - try { - fetch(); - } catch (Exception e) { - log.error("Failed to fetch remote repository", e); - } - }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { + ObjectNode versionInfo = JacksonUtil.newObjectNode(); + versionInfo.set("versionName", new TextNode(version.getName())); + versionInfo.set("versionId", new TextNode(version.getId())); + attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, + List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); } - private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Version not found"); - } - } - @Override - public List listAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptyList()); + public List listBranches(TenantId tenantId) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listBranches(); } - private void checkBranch(TenantId tenantId, String branch) { - if (!listAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to the branch"); - } - } + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); } - private void initRepository(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { - this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(Path.of(settings.getRepositoryDirectory())); - this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), - settings.getUsername(), settings.getPassword()); - } - } finally { - repositoryLock.writeLock().unlock(); + private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); } + repositories.put(tenantId, repository); } - private void clearRepository() throws IOException { - repositoryLock.writeLock().lock(); - try { - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repository = null; - } - } finally { - repositoryLock.writeLock().unlock(); + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); } } @SneakyThrows @Override - public void saveSettings(EntitiesVersionControlSettings settings) { - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .orElseGet(() -> { - AdminSettings newAdminSettings = new AdminSettings(); - newAdminSettings.setKey(SETTINGS_KEY); - return newAdminSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - - repositoryLock.writeLock().lock(); - try { - clearRepository(); - initRepository(settings); - } finally { - repositoryLock.writeLock().unlock(); - } + public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { + attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) + )).get(); - if (settings.getFetchPeriod() != 0) { - fetchTask.cancel(true); - fetchTask = scheduleFetch(settings.getFetchPeriod()); - } + clearRepository(tenantId); + initRepository(tenantId, settings); } + @SneakyThrows @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) .orElse(null); } @@ -387,7 +474,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 435c661b02..03343ec6f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -20,19 +20,18 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; import java.util.List; public interface EntitiesVersionControlService { - EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; - - EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; @@ -42,23 +41,19 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List listAllowedBranches(TenantId tenantId); + List listBranches(TenantId tenantId) throws Exception; - void saveSettings(EntitiesVersionControlSettings settings); + void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(); + EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 06e0a9c7aa..602f5dba4f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -17,18 +17,10 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; -import java.util.List; -import java.util.Map; -import java.util.UUID; - @Data public class EntitiesVersionControlSettings { - private String repositoryDirectory; private String repositoryUri; private String username; private String password; - - private int fetchPeriod; - - private Map> tenantsAllowedBranches; + private String defaultBranch; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java new file mode 100644 index 0000000000..ac13a7b8bf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; + +@Data +public class VersionCreationResult { + private EntityVersion version; + private int added; + private int modified; + private int removed; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java index 4cf0ee2b0f..cf8b588008 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java @@ -20,14 +20,14 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; @Data @AllArgsConstructor @NoArgsConstructor @Builder -public class EntityVersionLoadResult { - private ExportableEntity newEntityVersion; - private ExportableEntity previousEntityVersion; +public class VersionLoadResult { private EntityType entityType; + private int created; + private int updated; + private int deleted; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java index d71ddc8720..835f684021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java @@ -13,26 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityFilter; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityFilterExportRequest extends ExportRequest { +public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { private EntityFilter filter; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_FILTER; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java index 946676cf65..6386e9299a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java @@ -13,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityDataQuery; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityQueryExportRequest extends ExportRequest { +public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { private EntityDataQuery query; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_QUERY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java index 01309a421b..1f3e1e376f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java @@ -13,23 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityListExportRequest extends ExportRequest { +public class EntityListVersionCreateConfig extends VersionCreateConfig { private List entitiesIds; @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_LIST; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_LIST; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index fd2f659b70..6a8d59b876 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; @@ -21,18 +21,18 @@ import org.thingsboard.server.common.data.EntityType; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityTypeExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateConfig extends VersionCreateConfig { private EntityType entityType; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_TYPE; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_TYPE; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java index fe07ba7534..ab23f1175e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; -@EqualsAndHashCode(callSuper = true) @Data -public class SingleEntityExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateConfig extends VersionCreateConfig { private EntityId entityId; @Override - public ExportRequestType getType() { - return ExportRequestType.SINGLE_ENTITY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.SINGLE_ENTITY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index af0f161227..31b7b46f47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -23,17 +23,17 @@ import lombok.Data; @JsonTypeInfo(use = Id.NAME, property = "type") @JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = CustomEntityFilterExportRequest.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = CustomEntityQueryExportRequest.class) + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) }) @Data -public abstract class ExportRequest { +public abstract class VersionCreateConfig { - private EntityExportSettings exportSettings; + private boolean saveRelations; - public abstract ExportRequestType getType(); + public abstract VersionCreateConfigType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java index c36ac567e3..b2fb251d6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; -public enum ExportRequestType { +public enum VersionCreateConfigType { SINGLE_ENTITY, ENTITY_LIST, ENTITY_TYPE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 842f6ddd4e..47b07af21c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -13,12 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; +import java.util.List; + @Data -public class EntityVersionLoadSettings { - private boolean loadRelations; - private boolean findExistingEntityByName; +public class VersionCreateRequest { + + private String versionName; + private String branch; + + private List configs; + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java new file mode 100644 index 0000000000..f210dc9e0c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -0,0 +1,12 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { + + private boolean removeNonexistentLocalEntities; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java new file mode 100644 index 0000000000..6a49c45c2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadRequest extends VersionLoadRequest { + + private Map configs; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java new file mode 100644 index 0000000000..f7d99a0981 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class EntityVersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java new file mode 100644 index 0000000000..f5336b55d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionLoadRequest extends VersionLoadRequest { + + private EntityId externalEntityId; + + private EntityVersionLoadConfig config; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 1fc36463ba..660f5dc18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @Data -public class EntityVersionSaveSettings { - private boolean saveRelations; +public abstract class VersionLoadRequest { + + private String branch; + private String versionId; + + public abstract VersionLoadRequestType getType(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java new file mode 100644 index 0000000000..178d49668f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +public enum VersionLoadRequestType { + SINGLE_ENTITY, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 205bbca50e..14cdbf7b2c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -24,6 +24,7 @@ import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -33,18 +34,18 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -54,6 +55,8 @@ public class GitRepository { @Getter private final String directory; + @Getter + private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -61,21 +64,20 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(String uri, String directory, - String username, String password) throws GitAPIException { + public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() .setURI(uri) - .setDirectory(new java.io.File(directory)) + .setDirectory(directory) .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new GitRepository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); } - public static GitRepository open(String directory, String username, String password) throws IOException { - Git git = Git.open(new java.io.File(directory)); - return new GitRepository(git, newCredentialsProvider(username, password), directory); + public static GitRepository open(File directory, String username, String password) throws IOException { + Git git = Git.open(directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); } @@ -180,9 +182,17 @@ public class GitRepository { execute(git.clean()); } - - public Commit commit(String message, String filesPattern) throws GitAPIException { + public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); + } + + public Status status() throws GitAPIException { + org.eclipse.jgit.api.Status status = execute(git.status()); + return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + } + + public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); @@ -256,4 +266,11 @@ public class GitRepository { private final String authorName; } + @Data + public static class Status { + private final Set added; + private final Set modified; + private final Set removed; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index b5c5a5aab8..988bae7146 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -64,13 +64,13 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.nio.ByteBuffer; import java.util.Arrays; @@ -339,17 +339,17 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(entityId); exportRequest.setExportSettings(new EntityExportSettings()); return (EntityExportData) exportEntities(exportRequest).get(0); } - protected List> exportEntities(ExportRequest exportRequest) throws Exception { + protected List> exportEntities(VersionCreateConfig exportRequest) throws Exception { return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>() {}); } - protected List> exportEntities(List exportRequests) throws Exception { + protected List> exportEntities(List exportRequests) throws Exception { return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>() {}); } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index bd237e39d0..faa7f3a318 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; @@ -50,18 +49,18 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.ota.OtaPackageStateService; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.util.ArrayList; import java.util.List; @@ -294,11 +293,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp dashboard.setConfiguration(dashboardConfiguration); dashboard = dashboardService.saveDashboard(dashboard); - EntityTypeExportRequest assetsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig assetsExportRequest = new EntityTypeVersionCreateConfig(); assetsExportRequest.setEntityType(EntityType.ASSET); assetsExportRequest.setPageSize(10); assetsExportRequest.setExportSettings(new EntityExportSettings()); - EntityTypeExportRequest dashboardsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig dashboardsExportRequest = new EntityTypeVersionCreateConfig(); dashboardsExportRequest.setEntityType(EntityType.DASHBOARD); dashboardsExportRequest.setPageSize(10); dashboardsExportRequest.setExportSettings(new EntityExportSettings()); @@ -365,7 +364,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Customer 1 - Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), ruleChain.getId(), deviceProfile.getId(), dashboard.getId())); List> exportDataList = exportEntities(exportRequest); @@ -418,7 +417,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -464,7 +463,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -506,7 +505,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device1 = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(asset.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -538,7 +537,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(device.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -568,7 +567,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin1(); DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(defaultDeviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); List> exportDataList = exportEntities(exportRequest); @@ -613,7 +612,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testEntityTypeExportRequest(ExportableEntity entity) throws Exception { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entity.getId().getEntityType()); @@ -626,11 +625,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testCustomEntityFilterExportRequest(ExportableEntity entity) throws Exception { - CustomEntityFilterExportRequest exportRequest = new CustomEntityFilterExportRequest(); + EntitiesByCustomFilterVersionCreateConfig exportRequest = new EntitiesByCustomFilterVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); - EntityListFilter filter = new EntityListFilter(); + org.thingsboard.server.common.data.query.EntityListFilter filter = new org.thingsboard.server.common.data.query.EntityListFilter(); filter.setEntityType(entity.getId().getEntityType()); filter.setEntityList(List.of(entity.getId().toString())); exportRequest.setFilter(filter); @@ -652,10 +651,10 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Asset tenantAsset = createAsset(tenantId1, null, "A", "Tenant asset 1"); Asset customerAsset = createAsset(tenantId1, customer.getId(), "A", "Customer asset 1"); - List exportRequests = new ArrayList<>(); + List exportRequests = new ArrayList<>(); for (EntityType entityType : Set.of(EntityType.DEVICE, EntityType.ASSET)) { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entityType); @@ -685,7 +684,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), deviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 18444bb5f3..9240838543 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public abstract class DaoUtil { @@ -51,7 +53,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap()); } - public static Pageable toPageable(PageLink pageLink, Map columnMap) { + public static Pageable toPageable(PageLink pageLink, Map columnMap) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(pageLink.getSortOrder(), columnMap)); } @@ -59,7 +61,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } - public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(sortOrders, columnMap)); } @@ -108,4 +110,18 @@ public abstract class DaoUtil { return ids; } + public static void processInBatches(Function> finder, int batchSize, Consumer processor) { + PageLink pageLink = new PageLink(batchSize); + PageData batch; + + boolean hasNextBatch; + do { + batch = finder.apply(pageLink); + batch.getData().forEach(processor); + + hasNextBatch = batch.hasNext(); + pageLink = pageLink.nextPageLink(); + } while (hasNextBatch); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index cd534f24e6..aaa35e109a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -16,6 +16,8 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import java.util.UUID; @@ -25,4 +27,6 @@ public interface ExportableEntityDao> extends Dao< default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } + PageData findByTenantId(UUID tenantId, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index bcfdaae965..8879809ddb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -219,6 +219,11 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return findAssetsByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findAssetsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index c6ba310793..034431157a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -80,6 +80,11 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findCustomersByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 4a870b2a21..847314f1a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -31,4 +33,6 @@ public interface DashboardRepository extends JpaRepository findByTenantIdAndTitle(UUID tenantId, String title); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 6ff9fc6bfb..8fa0cbdc50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -21,6 +21,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -58,6 +60,11 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(dashboardRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + @Override public List findByTenantIdAndTitle(UUID tenantId, String title) { return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 93cd857660..d4663f60a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -313,6 +313,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return findDeviceByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findDevicesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index cae94cfb73..4cab209f49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -121,6 +121,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findDeviceProfiles(TenantId.fromUUID(tenantId), pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 0d5fea6f98..2703c99fba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -114,6 +114,11 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; From 2302213b3c8f288fa8284973e668285509f5e671 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 17:03:23 +0300 Subject: [PATCH 058/178] Refactor version create request structures --- .../DefaultEntitiesVersionControlService.java | 81 ++++++++++--------- ...tiesByCustomFilterVersionCreateConfig.java | 36 --------- ...itiesByCustomQueryVersionCreateConfig.java | 36 --------- .../create/EntityListVersionCreateConfig.java | 33 -------- .../EntityListVersionCreateRequest.java | 21 +++++ .../create/EntityTypeVersionCreateConfig.java | 38 --------- .../EntityTypeVersionCreateRequest.java | 20 +++++ .../MultipleEntitiesVersionCreateConfig.java | 10 +++ .../SingleEntityVersionCreateConfig.java | 33 -------- .../SingleEntityVersionCreateRequest.java | 19 +++++ .../vc/data/request/create/SyncStrategy.java | 6 ++ .../request/create/VersionCreateConfig.java | 16 ---- .../create/VersionCreateConfigType.java | 25 ------ .../request/create/VersionCreateRequest.java | 6 +- .../create/VersionCreateRequestType.java | 7 ++ .../load/EntityTypeVersionLoadConfig.java | 2 +- 16 files changed, 127 insertions(+), 262 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 919960ce62..f6434daecc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -65,6 +65,7 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; @@ -152,23 +153,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont if (repository.listBranches().contains(request.getBranch())) { repository.checkout(request.getBranch()); repository.merge(request.getBranch()); - } else { // FIXME [viacheslav]: rollback orphan branch on failure + } else { // TODO [viacheslav]: rollback orphan branch on failure repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - for (VersionCreateRequest.Config config : request.getConfigs()) { + for (VersionCreateConfig config : request.getConfigs()) { EntityExportSettings exportSettings = EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build(); List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities()) { + for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); entityDataList.add(entityData); } - if (config.isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() + if (config instanceof MultipleEntitiesVersionCreateConfig && + ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? .map(EntityExportData::getEntityType) .distinct() .forEach(entityType -> { @@ -186,7 +188,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } } - // TODO [viacheslav]: find with pagination repository.add("."); @@ -206,18 +207,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { - switch (entityFilter.getType()) { + private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { + switch (config.getType()) { case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; return List.of(filter.getEntityId()); } case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; return filter.getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); @@ -228,7 +229,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); EntityDataPageLink pageLink = new EntityDataPageLink(); @@ -243,7 +244,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() .stream().map(EntityData::getEntityId) @@ -341,35 +342,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont boolean removeNonexistentLocalEntities = false; if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); - - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); } - // need to delete in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); - } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); - } + } return new ArrayList<>(results.values()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java deleted file mode 100644 index 835f684021..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityFilter; - -import java.util.UUID; - -@Data -public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { - - private EntityFilter filter; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java deleted file mode 100644 index 6386e9299a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityDataQuery; - -import java.util.UUID; - -@Data -public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { - - private EntityDataQuery query; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java deleted file mode 100644 index 1f3e1e376f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -public class EntityListVersionCreateConfig extends VersionCreateConfig { - - private List entitiesIds; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java new file mode 100644 index 0000000000..fadd9b1fd9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -0,0 +1,21 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityListVersionCreateRequest extends VersionCreateRequest { + + private List entitiesIds; + private MultipleEntitiesVersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_LIST; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java deleted file mode 100644 index 6a8d59b876..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.EntityType; - -import java.util.UUID; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionCreateConfig extends VersionCreateConfig { - - private EntityType entityType; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_TYPE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java new file mode 100644 index 0000000000..0fd1d60667 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateRequest extends VersionCreateRequest { + + private Map entityTypes; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java new file mode 100644 index 0000000000..f6d852875d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -0,0 +1,10 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { + private SyncStrategy syncStrategy; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java deleted file mode 100644 index ab23f1175e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SingleEntityVersionCreateConfig extends VersionCreateConfig { - - private EntityId entityId; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.SINGLE_ENTITY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java new file mode 100644 index 0000000000..31bdeca658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -0,0 +1,19 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateRequest extends VersionCreateRequest { + + private EntityId entityId; + private VersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java new file mode 100644 index 0000000000..6216ad7658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum SyncStrategy { + MERGE, + OVERWRITE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 31b7b46f47..28f3232cb9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -15,25 +15,9 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import lombok.Data; -@JsonTypeInfo(use = Id.NAME, property = "type") -@JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) -}) @Data public abstract class VersionCreateConfig { - private boolean saveRelations; - - public abstract VersionCreateConfigType getType(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java deleted file mode 100644 index b2fb251d6a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -public enum VersionCreateConfigType { - SINGLE_ENTITY, - ENTITY_LIST, - ENTITY_TYPE, - - CUSTOM_ENTITY_FILTER, - CUSTOM_ENTITY_QUERY -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 47b07af21c..e11e2f85a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -17,14 +17,12 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import java.util.List; - @Data -public class VersionCreateRequest { +public abstract class VersionCreateRequest { private String versionName; private String branch; - private List configs; + public abstract VersionCreateRequestType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java new file mode 100644 index 0000000000..a5d0447b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -0,0 +1,7 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum VersionCreateRequestType { + SINGLE_ENTITY, + ENTITY_LIST, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index f210dc9e0c..b5ee5ddd50 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -7,6 +7,6 @@ import lombok.EqualsAndHashCode; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { - private boolean removeNonexistentLocalEntities; + private boolean removeOtherEntities; } From 5a43788b2498700135b4df4f6f1e5d7ed81b4f5c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 15:32:07 +0300 Subject: [PATCH 059/178] VC refactoring; Swagger examples --- .../EntitiesVersionControlController.java | 141 ++++++- .../DefaultEntitiesExportImportService.java | 40 +- .../EntitiesExportImportService.java | 5 + .../DefaultExportableEntitiesService.java | 46 --- .../exporting/ExportableEntitiesService.java | 2 - .../DefaultEntitiesVersionControlService.java | 386 +++++++++--------- .../sync/vc/data/VersionCreationResult.java | 15 + .../sync/vc/data/VersionedEntityInfo.java | 1 - .../EntityListVersionCreateRequest.java | 15 + .../EntityTypeVersionCreateRequest.java | 15 + .../MultipleEntitiesVersionCreateConfig.java | 17 +- .../SingleEntityVersionCreateRequest.java | 15 + .../vc/data/request/create/SyncStrategy.java | 15 + .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 9 + .../create/VersionCreateRequestType.java | 15 + .../load/EntityTypeVersionLoadConfig.java | 17 +- .../load/EntityTypeVersionLoadRequest.java | 17 +- .../request/load/EntityVersionLoadConfig.java | 11 - .../load/SingleEntityVersionLoadRequest.java | 17 +- .../data/request/load/VersionLoadConfig.java | 26 ++ .../data/request/load/VersionLoadRequest.java | 9 + .../request/load/VersionLoadRequestType.java | 15 + .../server/utils/GitRepository.java | 4 +- 24 files changed, 580 insertions(+), 275 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0625dbeb38..e7df5acdc3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -44,6 +45,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; + @RestController @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -53,6 +56,62 @@ public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"saveRelations\": true\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_LIST:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_LIST\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entitiesIds\": [\n" + + " {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " {\n" + + " \"entityType\": \"DEVICE_PROFILE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " }\n" + + " ],\n" + + " \"config\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " },\n" + + " \"DEVICE_PROFILE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"OVERWRITE\"\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/version") public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -64,6 +123,13 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profile 1 version 1.0\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") public List listEntityVersions(@PathVariable String branch, @PathVariable EntityType entityType, @@ -76,6 +142,13 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profiles from dev\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}") public List listEntityTypeVersions(@PathVariable String branch, @PathVariable EntityType entityType) throws ThingsboardException { @@ -86,6 +159,21 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" + + " \"name\": \"Device 1 removed\"\n" + + " },\n" + + " {\n" + + " \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \"name\": \"Device profiles added\"\n" + + " },\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Devices added\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}") public List listVersions(@PathVariable String branch) throws ThingsboardException { try { @@ -118,6 +206,38 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + " \n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \n" + + " \"externalEntityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false,\n" + + " \"removeOtherEntities\": true\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/entity") public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -139,8 +259,21 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + - "") + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"name\": \"master\",\n" + + " \"default\": true\n" + + " },\n" + + " {\n" + + " \"name\": \"dev\",\n" + + " \"default\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"dev-2\",\n" + + " \"default\": false\n" + + " }\n" + + "]\n\n```") @GetMapping("/branches") public List listBranches() throws ThingsboardException { try { @@ -161,7 +294,7 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + @@ -177,7 +310,7 @@ public class EntitiesVersionControlController extends BaseController { } } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 608a256514..108c2c0b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -70,15 +70,37 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } + @Override + public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + + EntityType entityType = exportData.getEntityType(); + EntityImportService> importService = getImportService(entityType); + + EntityImportResult importResult = importService.importEntity(user, exportData, importSettings); + + if (saveReferences) { + importResult.getSaveReferencesCallback().run(); + } + if (sendEvents) { + importResult.getSendEventsCallback().run(); + } + + return importResult; + } + @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixOrder(exportDataList); + fixDataOrderForImport(exportDataList); List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { - EntityImportResult importResult = importEntity(user, exportData, importSettings); + EntityImportResult importResult = importEntity(user, exportData, importSettings, false, false); importResults.add(importResult); } @@ -103,21 +125,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } - private void fixOrder(List> exportDataList) { + @Override + public void fixDataOrderForImport(List> exportDataList) { exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); } - private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { - if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { - throw new DataValidationException("Invalid entity data"); - } - - EntityType entityType = exportData.getEntityType(); - EntityImportService> importService = getImportService(entityType); - - return importService.importEntity(user, exportData, importSettings); - } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index 5c29feb57c..049cf4390a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -30,6 +30,11 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException; + List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; + void fixDataOrderForImport(List> exportDataList); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index 4d772ad935..dafd68ba7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -17,11 +17,9 @@ package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; @@ -47,12 +45,6 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -145,44 +137,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } - @Transactional(readOnly = true, timeout = 40) - @Override - public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { - switch (request.getType()) { - case SINGLE_ENTITY: { - return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); - } - case ENTITY_LIST: { - return ((EntityListVersionCreateConfig) request).getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(exportRequest.getEntityType()); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; - EntityFilter filter = exportRequest.getFilter(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; - EntityDataQuery query = exportRequest.getQuery(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByQuery(tenantId, customerId, query); - } - default: { - throw new IllegalArgumentException("Export request is not supported"); - } - } - } - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 043ec91e20..66c9079148 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -39,8 +39,6 @@ public interface ExportableEntitiesService { , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); - List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: - void deleteByTenantIdAndId(TenantId tenantId, I id); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index f6434daecc..0bdedb5e4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,16 +18,14 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -40,12 +38,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; @@ -65,17 +63,18 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; +import org.thingsboard.server.utils.ThrowingRunnable; import java.io.File; import java.io.IOException; @@ -108,10 +107,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final AttributesService attributesService; private final EntityService entityService; private final TenantDao tenantDao; + private final TransactionTemplate transactionTemplate; // TODO [viacheslav]: concurrency private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}") + @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; private static final String SETTINGS_KEY = "vc"; @@ -146,33 +146,26 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - repository.getLock().writeLock().lock(); - - try { - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } - - for (VersionCreateConfig config : request.getConfigs()) { - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(config.isSaveRelations()) - .build(); - List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); - } + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // TODO [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } - if (config instanceof MultipleEntitiesVersionCreateConfig && - ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? - .map(EntityExportData::getEntityType) - .distinct() + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + break; + } + case ENTITY_LIST: { + EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; + if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { + versionCreateRequest.getEntitiesIds().stream() + .map(EntityId::getEntityType).distinct() .forEach(entityType -> { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); @@ -181,76 +174,70 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } }); } - - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { + saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); } + break; } + case ENTITY_TYPE: { + EntityTypeVersionCreateRequest versionCreateRequest = (EntityTypeVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + break; + } + } - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); + repository.add("."); - result.setVersion(toVersion(commit)); - return result; - } finally { - repository.getLock().writeLock().unlock(); - } - } + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { - switch (config.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; - return List.of(filter.getEntityId()); - } - case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; - return filter.getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; - EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + GitRepository.Commit commit = repository.commit(request.getVersionName()); + repository.push(); - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(filter.getEntityType()); + result.setVersion(toVersion(commit)); + return result; + } - newFilter.setFilter(entityTypeFilter); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; - EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); - - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - newFilter.setQuery(query); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); - return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() - .stream().map(EntityData::getEntityId) - .collect(Collectors.toList()); - } - } + private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build()); + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } @@ -271,14 +258,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listVersions(TenantId tenantId, String branch, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); } @@ -294,23 +276,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); - - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - info.setEntityName(entityData.getEntity().getName()); - return info; - }) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); } @@ -318,81 +292,117 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - List> entityDataList = new ArrayList<>(); - EntityVersion version; - repository.getLock().readLock().lock(); - try { - version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { - EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); - entityDataList.add(entityData); - } - } finally { - repository.getLock().readLock().unlock(); - } - - EntityImportSettings importSettings = EntityImportSettings.builder() - .updateRelations(settings.isLoadRelations()) - .findExistingByName(settings.isFindExistingEntityByName()) - .build(); - // FIXME [viacheslav]: do evrth in transaction - List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - - Map results = new HashMap<>(); + EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - boolean removeNonexistentLocalEntities = false; - if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + result.getSaveReferencesCallback().run(); + return result; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + try { + importResult.getSendEventsCallback().run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + return List.of(VersionLoadResult.builder() + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } + case ENTITY_TYPE: { + EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; + return transactionTemplate.execute(status -> { + Map results = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + // order entity types.. + // or what + versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + AtomicInteger deleted = new AtomicInteger(); + + Set remoteEntities; + try { + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + .map(VersionedEntityInfo::getExternalId) + .collect(Collectors.toSet()); + for (EntityId externalEntityId : remoteEntities) { + EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (config.isRemoveOtherEntities()) { DaoUtil.processInBatches(pageLink -> { return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { try { exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); } catch (ThingsboardException e) { throw new RuntimeException(e); } - // need to delete in a specific order? + // need to delete entity types in a specific order? exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); deleted.getAndIncrement(); } }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); + } + + results.put(entityType, VersionLoadResult.builder() + .created(created.get()) + .updated(updated.get()) + .deleted(deleted.get()) + .build()); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return new ArrayList<>(results.values()); + }); } - - return new ArrayList<>(results.values()); - } - - @SneakyThrows - private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { - GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } finally { - repository.getLock().readLock().unlock(); + default: + throw new IllegalArgumentException("Unsupported version load request"); } } - private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { - ObjectNode versionInfo = JacksonUtil.newObjectNode(); - versionInfo.set("versionName", new TextNode(version.getName())); - versionInfo.set("versionId", new TextNode(version.getId())); - attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, - List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); + private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); + + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); } @@ -417,12 +427,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - if (Files.exists(repositoryDirectory)) { - repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - } + FileUtils.forceDelete(repositoryDirectory.toFile()); + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); repositories.put(tenantId, repository); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java index ac13a7b8bf..9194c67272 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 97bb6ada12..0a65297892 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -21,6 +21,5 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class VersionedEntityInfo { private EntityId externalId; - private String entityName; // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java index fadd9b1fd9..59cf7d0176 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java index 0fd1d60667..f53e563e28 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java index f6d852875d..0a38e3c922 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @@ -5,6 +20,6 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { +public class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { private SyncStrategy syncStrategy; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java index 31bdeca658..4d03404199 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java index 6216ad7658..43ec873f22 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum SyncStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 28f3232cb9..d4f8354df4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -18,6 +18,6 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @Data -public abstract class VersionCreateConfig { +public class VersionCreateConfig { private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index e11e2f85a3..f4ba122f6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateRequest.class) +}) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index a5d0447b40..add3591fa2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index b5ee5ddd50..27597f5e87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -5,7 +20,7 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { +public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java index 6a49c45c2c..0ce902c83e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ import java.util.Map; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadRequest extends VersionLoadRequest { - private Map configs; + private Map entityTypes; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java deleted file mode 100644 index f7d99a0981..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.thingsboard.server.service.sync.vc.data.request.load; - -import lombok.Data; - -@Data -public class EntityVersionLoadConfig { - - private boolean loadRelations; - private boolean findExistingEntityByName; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java index f5336b55d1..eb4bb1bdc1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ public class SingleEntityVersionLoadRequest extends VersionLoadRequest { private EntityId externalEntityId; - private EntityVersionLoadConfig config; + private VersionLoadConfig config; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java new file mode 100644 index 0000000000..6ead76f105 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class VersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 660f5dc18d..5a24bc78e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.load; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +import static com.fasterxml.jackson.annotation.JsonSubTypes.Type; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionLoadRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionLoadRequest.class) +}) @Data public abstract class VersionLoadRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java index 178d49668f..6a59cba676 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; public enum VersionLoadRequestType { diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 14cdbf7b2c..2cb01e6a8b 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -55,8 +55,6 @@ public class GitRepository { @Getter private final String directory; - @Getter - private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -88,7 +86,7 @@ public class GitRepository { public void checkout(String branch) throws GitAPIException { execute(git.checkout() - .setName(branch)); + .setName("origin/" + branch)); } public void merge(String branch) throws IOException, GitAPIException { From 9b209e407553509821ce8b3ad010d6624f3a1ea6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 18:00:53 +0300 Subject: [PATCH 060/178] Refactor version create request; fixes for git api usage --- .../EntitiesVersionControlController.java | 45 +++------ .../DefaultEntitiesVersionControlService.java | 96 ++++++++++--------- .../EntityListVersionCreateRequest.java | 36 ------- .../create/EntityTypeVersionCreateConfig.java | 4 +- .../request/create/VersionCreateRequest.java | 3 +- .../create/VersionCreateRequestType.java | 1 - .../server/utils/GitRepository.java | 30 +++--- 7 files changed, 86 insertions(+), 129 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..7c2c2ac483 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController { " \"saveRelations\": true\n" + " }\n" + "}\n```" + NEW_LINE + - "ENTITY_LIST:" + NEW_LINE + + "COMPLEX:" + NEW_LINE + "```\n{\n" + - " \"type\": \"ENTITY_LIST\",\n" + + " \"type\": \"COMPLEX\",\n" + "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + - "\n" + - " \"entitiesIds\": [\n" + - " {\n" + - " \"entityType\": \"DEVICE\",\n" + - " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + - " },\n" + - " {\n" + - " \"entityType\": \"DEVICE_PROFILE\",\n" + - " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + - " }\n" + - " ],\n" + - " \"config\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + - " }\n" + - "}\n```" + NEW_LINE + - "ENTITY_TYPE:" + NEW_LINE + - "```\n{\n" + - " \"type\": \"ENTITY_TYPE\",\n" + - "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + " \"entityTypes\": {\n" + " \"DEVICE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true\n" + " },\n" + " \"DEVICE_PROFILE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"OVERWRITE\"\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + " }\n" + " }\n" + "}\n```") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..7fa99d9e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; @@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; @@ -88,6 +90,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont repository.fetch(); if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); + repository.checkout("origin/" + request.getBranch(), false); + try { + repository.checkout(request.getBranch(), true); + } catch (RefAlreadyExistsException e) { + repository.checkout(request.getBranch(), false); + } repository.merge(request.getBranch()); } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + try { + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } } switch (request.getType()) { @@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } - case ENTITY_LIST: { - EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; - if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { - versionCreateRequest.getEntitiesIds().stream() - .map(EntityId::getEntityType).distinct() - .forEach(entityType -> { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { - saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); - } - break; - } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); } catch (IOException e) { @@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); - try { - saveEntityData(user, repository, entityId, config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + } catch (Exception e) { + throw new RuntimeException(e); + } } - }); + } + }); break; } @@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); @@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - clearRepository(tenantId); initRepository(tenantId, settings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java deleted file mode 100644 index adeac70e89..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityListVersionCreateRequest extends VersionCreateRequest { - - private List entitiesIds; - private EntityTypeVersionCreateConfig config; - - @Override - public VersionCreateRequestType getType() { - return VersionCreateRequestType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index 1c27d9d2f1..0cc890c0d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; import java.util.List; +import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) @@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig { //optional private SyncStrategy syncStrategy; - private List entityIds; + private List entityIds; private boolean allEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 10f093d26d..a99bd56d7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -23,8 +23,7 @@ import lombok.Data; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), - @Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class) + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) }) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index fea8b6fec7..aacc57f3b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, - ENTITY_LIST, COMPLEX } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..ac8df955d6 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -84,9 +81,15 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch) throws GitAPIException { + public void checkout(String branch, boolean createBranch) throws GitAPIException { execute(git.checkout() - .setName("origin/" + branch)); + .setCreateBranch(createBranch) + .setName(branch)); + } + + public void reset() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); } public void merge(String branch) throws IOException, GitAPIException { @@ -170,14 +173,15 @@ public class GitRepository { public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) + .setForced(true) .setName(name)); - Set uncommittedChanges = git.status().call().getUncommittedChanges(); - if (!uncommittedChanges.isEmpty()) { - RmCommand rm = git.rm(); - uncommittedChanges.forEach(rm::addFilepattern); - execute(rm); - } - execute(git.clean()); +// Set uncommittedChanges = git.status().call().getUncommittedChanges(); +// if (!uncommittedChanges.isEmpty()) { +// RmCommand rm = git.rm(); +// uncommittedChanges.forEach(rm::addFilepattern); +// execute(rm); +// } +// execute(git.clean()); } public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] From 7ec45fd5f1d602f23fde6225b7c4ee7b77e66f52 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 10:40:02 +0300 Subject: [PATCH 061/178] Rename Data structures. Create separate module. --- application/pom.xml | 4 + .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesVersionControlController.java | 14 +-- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 20 ++-- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 2 +- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 5 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 3 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +-- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../DefaultEntitiesVersionControlService.java | 42 ++++----- .../vc/EntitiesVersionControlService.java | 14 +-- .../common/data/sync}/JsonTbEntity.java | 2 +- .../common/data/sync}/ThrowingRunnable.java | 2 +- .../data/sync/ie}/DeviceExportData.java | 2 +- .../data/sync/ie}/EntityExportData.java | 4 +- .../data/sync/ie}/EntityExportSettings.java | 2 +- .../data/sync/ie}/EntityImportResult.java | 6 +- .../data/sync/ie}/EntityImportSettings.java | 2 +- .../data/sync/ie}/RuleChainExportData.java | 2 +- .../vc}/EntitiesVersionControlSettings.java | 2 +- .../common/data/sync/vc}/EntityVersion.java | 2 +- .../data/sync/vc}/VersionCreationResult.java | 2 +- .../data/sync/vc}/VersionLoadResult.java | 2 +- .../data/sync/vc}/VersionedEntityInfo.java | 2 +- .../create/ComplexVersionCreateRequest.java | 2 +- .../create/EntityTypeVersionCreateConfig.java | 2 +- .../SingleEntityVersionCreateRequest.java | 2 +- .../sync/vc}/request/create/SyncStrategy.java | 2 +- .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 2 +- .../create/VersionCreateRequestType.java | 2 +- .../load/EntityTypeVersionLoadConfig.java | 2 +- .../load/EntityTypeVersionLoadRequest.java | 2 +- .../load/SingleEntityVersionLoadRequest.java | 2 +- .../vc}/request/load/VersionLoadConfig.java | 2 +- .../vc}/request/load/VersionLoadRequest.java | 2 +- .../request/load/VersionLoadRequestType.java | 2 +- common/pom.xml | 1 + common/version-control/pom.xml | 94 +++++++++++++++++++ pom.xml | 5 + 61 files changed, 241 insertions(+), 141 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/DefaultEntitiesExportImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/EntitiesExportImportService.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/DefaultExportableEntitiesService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/ExportableEntitiesService.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/AbstractBulkImportService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/RuleChainImportService.java (95%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/JsonTbEntity.java (97%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/ThrowingRunnable.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/DeviceExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportResult.java (89%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/RuleChainExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntitiesVersionControlSettings.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntityVersion.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionCreationResult.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionLoadResult.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionedEntityInfo.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/ComplexVersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/EntityTypeVersionCreateConfig.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SingleEntityVersionCreateRequest.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SyncStrategy.java (90%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateConfig.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequestType.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/SingleEntityVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequestType.java (91%) create mode 100644 common/version-control/pom.xml diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..2ff2204b5a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 01401deebe..2a3268146f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,8 +51,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 61e44241d7..8b6f49c491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,8 +66,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8136b6d052..b3bbb03f4c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 7c2c2ac483..2ab75c565d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -32,13 +32,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index a2b71267b4..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,8 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index ee119c4dce..f0a7b6231e 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fea50538cc..b0fb2161e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,8 +28,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 108c2c0b1b..72e3d757be 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,15 +27,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 049cf4390a..5f5bed9dac 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index dafd68ba7b..18c5f5c7c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 5532f30125..9b251a3dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 66c9079148..742b993cfc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -25,9 +25,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; - -import java.util.List; public interface ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 46428a763f..80850cb462 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 69af613480..31bf768e90 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 93f9c51655..2e03b92a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index f40ee6884a..15c9c72a74 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index be6f62efe4..bf249937b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing; +package org.thingsboard.server.service.sync.ie.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 2be56ebe2a..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -66,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 9c0e6b6491..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 6347910cab..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 550d4a72f3..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index a39dcc7481..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 80a942c73f..fe6c3f1875 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ecfa7d3ab..203b0caa16 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index b69c5a0d5b..2a7570e644 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index b85e403f87..16ebdd580e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 112152365b..eb123a63af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index f97950aaf8..592fe4bc5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 993f9fa27a..1f1f15a74e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7fa99d9e6d..c22eb6a5a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -55,28 +55,28 @@ import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; import java.io.IOException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..e025f80fed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,13 +19,13 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index cfc38ff3c3..9d326246d4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; import com.fasterxml.jackson.annotation.JsonSubTypes; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java index 858ccdf422..1cb2ac8c74 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import org.thingsboard.server.common.data.exception.ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index 4be338d7cc..faf9280b93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 81e3b344d0..b361a302fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.utils.JsonTbEntity; +import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0fb318b5fb..051745c07f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 242856bfcb..ea69425a2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.utils.JsonTbEntity; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.JsonTbEntity; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 075a6896ef..3e18bc9f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java index e1d870473f..b176782f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ee8f5b40b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 3547f89e3c..8779d64ed3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java index 9194c67272..8cb09b1b30 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index cf8b588008..0039546fb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 0a65297892..163fe4c6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java index 3c34680f95..f8d7279f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java index 0cc890c0d7..92cab96354 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java index 4d03404199..507afd116e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java index 43ec873f22..baf24efeb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum SyncStrategy { MERGE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index d4f8354df4..a1aa6cd6d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java index a99bd56d7f..9a23838921 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java index aacc57f3b3..d57363cf7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 27597f5e87..837986f82d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java index 0ce902c83e..f2cb83f0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java index eb4bb1bdc1..cf31317b4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 6ead76f105..31b34dbcb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java index 5a24bc78e5..d9d1329c8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java index 6a59cba676..190e6871d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; public enum VersionLoadRequestType { SINGLE_ENTITY, diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..1b7194e029 --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.springframework + spring-core + + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.thingsboard.common + data + + + + + + + + + diff --git a/pom.xml b/pom.xml index e215e0719d..324de28f3a 100755 --- a/pom.xml +++ b/pom.xml @@ -866,6 +866,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache From c6ebf049a2dd7910293a3d728533686ec6bc663a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 14:46:54 +0300 Subject: [PATCH 062/178] Version Control Service refactoring --- application/pom.xml | 4 - .../EntitiesVersionControlController.java | 2 +- .../DefaultEntitiesVersionControlService.java | 211 ++------------ .../vc/EntitiesVersionControlService.java | 2 +- .../vc/LocalGitVersionControlService.java | 258 ++++++++++++++++++ .../src/main/resources/thingsboard.yml | 5 + common/version-control/pom.xml | 21 ++ .../sync/vc/DefaultGitRepositoryService.java | 239 ++++++++++++++++ .../service/sync/vc}/GitRepository.java | 2 +- .../service/sync/vc/GitRepositoryService.java | 48 ++++ .../sync/vc/GitVersionControlService.java | 60 ++++ .../server/service/sync/vc/PendingCommit.java | 36 +++ 12 files changed, 695 insertions(+), 193 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java rename {application/src/main/java/org/thingsboard/server/utils => common/version-control/src/main/java/org/thingsboard/server/service/sync/vc}/GitRepository.java (99%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java diff --git a/application/pom.xml b/application/pom.xml index 2ff2204b5a..37d782b3b0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,10 +345,6 @@ Java-WebSocket test - - org.eclipse.jgit - org.eclipse.jgit - diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 2ab75c565d..2794db90d4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController { @PathVariable EntityType entityType, @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c22eb6a5a5..e222d027e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.utils.GitRepository; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; @@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final GitVersionControlService gitService; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AttributesService attributesService; private final EntityService entityService; - private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; - // TODO [viacheslav]: concurrency - private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}/repositories") - private String repositoriesFolder; - - private static final String SETTINGS_KEY = "vc"; - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); - if (settings != null) { - try { - initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, 5, 5, TimeUnit.SECONDS); - } - + public static final String SETTINGS_KEY = "vc"; @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout("origin/" + request.getBranch(), false); - try { - repository.checkout(request.getBranch(), true); - } catch (RefAlreadyExistsException e) { - repository.checkout(request.getBranch(), false); - } - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } - } + + var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } + gitService.deleteAll(commit, entityType); } if (config.isAllEntities()) { @@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, 100, data -> { EntityId entityId = data.getEntityId(); try { - saveEntityData(user, repository, entityId, config); + saveEntityData(user, commit, entityId, config); } catch (Exception e) { throw new RuntimeException(e); } @@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); - - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); - - result.setVersion(toVersion(commit)); - return result; + return gitService.push(commit); } - private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + gitService.addToCommit(commit, entityData); } @Override public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + return gitService.listVersions(tenantId, branch, externalId); } @Override public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + return gitService.listVersions(tenantId, branch, entityType); } @Override public List listVersions(TenantId tenantId, String branch) throws Exception { - return listVersions(tenantId, branch, null); - } - - private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return gitService.listVersions(tenantId, branch); } - @Override - public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, null); + return gitService.listEntitiesAtVersion(tenantId, branch, versionId); } - private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - return info; - }) - .collect(Collectors.toList()); - } - - @Override public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); result.getSaveReferencesCallback().run(); return result; } catch (Exception e) { @@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Set remoteEntities; try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() .map(VersionedEntityInfo::getExternalId) .collect(Collectors.toSet()); for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); if (importResult.getOldEntity() == null) created.incrementAndGet(); else updated.incrementAndGet(); @@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); - + private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .findExistingByName(config.isFindExistingEntityByName()) @@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listBranches(TenantId tenantId) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listBranches(); - } - - - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - - private GitRepository checkRepository(TenantId tenantId) { - return Optional.ofNullable(repositories.get(tenantId)) - .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); - } - - private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - if (Files.exists(repositoryDirectory)) { - FileUtils.forceDelete(repositoryDirectory.toFile()); - } - - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - repositories.put(tenantId, repository); - } - - private void clearRepository(TenantId tenantId) throws IOException { - GitRepository repository = repositories.get(tenantId); - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repositories.remove(tenantId); - } + return gitService.listBranches(tenantId); } - @SneakyThrows @Override public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { @@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - initRepository(tenantId, settings); + gitService.initRepository(tenantId, settings); } - @SneakyThrows @Override public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - - private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; + return gitService.getSettings(tenantId); } - - private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); - String entityId = StringUtils.substringBetween(path, "/", ".json"); - return EntityIdFactory.getByTypeAndUuid(entityType, entityId); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index e025f80fed..f0d4169b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -41,7 +41,7 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java new file mode 100644 index 0000000000..b9a5a9260b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -0,0 +1,258 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.io.IOException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; + +@Slf4j +@RequiredArgsConstructor +@Service +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +public class LocalGitVersionControlService implements GitVersionControlService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final GitRepositoryService gitRepositoryService; + private final TenantDao tenantDao; + private final AttributesService attributesService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + @AfterStartUp + public void init() { + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } + } + }); + } + + @Override + @SneakyThrows + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) + .orElse(null); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + var pendingCommit = new PendingCommit(tenantId, request); + PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); + if (old != null) { + gitRepositoryService.abort(old); + } + gitRepositoryService.prepareCommit(pendingCommit); + return pendingCommit; + } finally { + lock.unlock(); + } + } + + @Override + public void deleteAll(PendingCommit commit, EntityType entityType) { + doInsideLock(commit, c -> { + try { + gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public void addToCommit(PendingCommit commit, EntityExportData> entityData) { + doInsideLock(commit, c -> { + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString()), entityDataJson); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + return executeInsideLock(commit, gitRepositoryService::push); + } + + @Override + public List listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, branch, (String) null); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + try { + return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + @Override + public List listBranches(TenantId tenantId) { + return gitRepositoryService.listBranches(tenantId); + } + + @Override + public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { + try { + String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, + getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private List listVersions(TenantId tenantId, String branch, String path) { + try { + return gitRepositoryService.listVersions(tenantId, branch, path); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private void doInsideLock(PendingCommit commit, Consumer r) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + r.accept(commit); + } finally { + lock.unlock(); + } + } + + private T executeInsideLock(PendingCommit commit, Function c) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + return c.apply(commit); + } finally { + lock.unlock(); + } + } + + private void checkCommit(PendingCommit commit) { + PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); + if (existing == null || !existing.getTxId().equals(commit.getTxId())) { + throw new ConcurrentModificationException(); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03dbb49763..654929f500 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1115,6 +1115,11 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + git: + service: "${JS_VC_GIT_SERVICE:local}" # local/remote + repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + management: endpoints: web: diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 1b7194e029..f29b7db81a 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -40,6 +40,23 @@ org.springframework spring-core + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + com.google.guava guava @@ -65,6 +82,10 @@ ch.qos.logback logback-classic + + org.eclipse.jgit + org.eclipse.jgit + org.springframework.boot spring-boot-starter-test diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..d22fcf5d94 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + @Value("${vc.git.repos-poll-interval:60}") + private long reposPollInterval; + + private ScheduledExecutorService scheduler; + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getRequest().getBranch(); + try { + repository.fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout("origin/" + branch, false); + try { + repository.checkout(branch, true); + } catch (RefAlreadyExistsException e) { + repository.checkout(branch, false); + } + repository.merge(branch); + } else { // TODO [viacheslav]: rollback orphan branch on failure + try { + repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); + repository.push(); + + result.setVersion(toVersion(gitCommit)); + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void abort(PendingCommit commit) { + //TODO: implement; + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + } + + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + } + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/utils/GitRepository.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index ac8df955d6..afebe1f7fc 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..2191f38930 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.io.IOException; +import java.util.List; + +public interface GitRepositoryService { + + void prepareCommit(PendingCommit pendingCommit); + + List listVersions(TenantId tenantId, String branch, String path) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java new file mode 100644 index 0000000000..e67a8cca5f --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlService { + + @SneakyThrows + EntitiesVersionControlSettings getSettings(TenantId tenantId); + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); + + void addToCommit(PendingCommit commit, EntityExportData> entityData); + + void deleteAll(PendingCommit pendingCommit, EntityType entityType); + + VersionCreationResult push(PendingCommit commit); + + List listVersions(TenantId tenantId, String branch); + + List listVersions(TenantId tenantId, String branch, EntityType entityType); + + List listVersions(TenantId tenantId, String branch, EntityId entityId); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + List listBranches(TenantId tenantId); + + EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..a30f7514a2 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final TenantId tenantId; + private final VersionCreateRequest request; + + public PendingCommit(TenantId tenantId, VersionCreateRequest request) { + this.txId = UUID.randomUUID(); + this.tenantId = tenantId; + this.request = request; + } +} From 1d6b9a5cbdc05c514787e6c6d795dc66549fcb73 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 16:43:01 +0300 Subject: [PATCH 063/178] Version control executor --- common/version-control/pom.xml | 27 ++++ .../sync/vc/DefaultGitRepositoryService.java | 1 - .../version-control/src/main/proto/vc.proto | 71 +++++++++ msa/pom.xml | 1 + msa/vc-executor/pom.xml | 136 ++++++++++++++++++ msa/vc-executor/src/main/conf/logback.xml | 43 ++++++ .../src/main/conf/tb-vc-executor.conf | 22 +++ ...oardVersionControlExecutorApplication.java | 47 ++++++ .../src/main/resources/logback.xml | 34 +++++ .../src/main/resources/tb-vc-executor.yml | 27 ++++ 10 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 common/version-control/src/main/proto/vc.proto create mode 100644 msa/vc-executor/pom.xml create mode 100644 msa/vc-executor/src/main/conf/logback.xml create mode 100644 msa/vc-executor/src/main/conf/tb-vc-executor.conf create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java create mode 100644 msa/vc-executor/src/main/resources/logback.xml create mode 100644 msa/vc-executor/src/main/resources/tb-vc-executor.yml diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index f29b7db81a..164640fe73 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,6 +86,21 @@ org.eclipse.jgit org.eclipse.jgit + + io.grpc + grpc-netty-shaded + provided + + + io.grpc + grpc-protobuf + provided + + + io.grpc + grpc-stub + provided + org.springframework.boot spring-boot-starter-test @@ -109,7 +124,19 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index d22fcf5d94..c93b4ae76d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -226,7 +226,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto new file mode 100644 index 0000000000..e8c7e09384 --- /dev/null +++ b/common/version-control/src/main/proto/vc.proto @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +option java_package = "org.thingsboard.server.gen.vc.v1"; +option java_multiple_files = true; +option java_outer_classname = "EdgeProtos"; + +package vc; + +// Interface exported by the ThingsBoard Core. +service TbGitRpcService { + + rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} + +} + + +/** + * Data Structures; + */ +message CommitRequestMsg { + string txId = 1; + PrepareMsg prepareMsg = 2; + AddMsg addMsg = 3; + DeleteMsg deleteMsg = 4; + PushMsg pushMsg = 5; + AbortMsg abortMsg = 6; +} + +message CommitResponseMsg { + string id = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string tenantId = 1; + string commitMsg = 2; + string branchName = 3; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} \ No newline at end of file diff --git a/msa/pom.xml b/msa/pom.xml index 106f2be2a3..888291502b 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -40,6 +40,7 @@ tb + vc-executor js-executor web-ui tb-node diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml new file mode 100644 index 0000000000..7e4df36117 --- /dev/null +++ b/msa/vc-executor/pom.xml @@ -0,0 +1,136 @@ + + + 4.0.0 + + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor + + ThingsBoard Version Control Executor + https://thingsboard.io + Project for ThingsBoard version control microservice + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-mqtt-transport + false + ${project.build.directory}/windows + ThingsBoard Version Control Executor Service + org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication + + + + + org.thingsboard.common + version-control + + + org.springframework.boot + spring-boot-starter-web + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml new file mode 100644 index 0000000000..d62cf2b3f5 --- /dev/null +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -0,0 +1,43 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/msa/vc-executor/src/main/conf/tb-vc-executor.conf b/msa/vc-executor/src/main/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..83287286bb --- /dev/null +++ b/msa/vc-executor/src/main/conf/tb-vc-executor.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java new file mode 100644 index 0000000000..2724a566ee --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.vc; /** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +public class ThingsboardVersionControlExecutorApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-vc-executor"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardVersionControlExecutorApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml new file mode 100644 index 0000000000..2834934ee0 --- /dev/null +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -0,0 +1,34 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml new file mode 100644 index 0000000000..1de47e2750 --- /dev/null +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# If you enabled process metrics you should also enable 'web-environment'. +spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" +# If you enabled process metrics you should set 'web-application-type' to 'servlet' value. +spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" + +server: + # Server bind address (has no effect if web-environment is disabled). + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port (has no effect if web-environment is disabled). + port: "${HTTP_BIND_PORT:8080}" + From 567ee5d75bb20b9a2803adbd506b5a8e23dfc450 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 18 May 2022 18:49:21 +0300 Subject: [PATCH 064/178] Refactor loadEntitiesVersion --- .../DefaultEntitiesExportImportService.java | 12 +- .../sync/ie/EntitiesExportImportService.java | 7 +- .../DefaultEntitiesVersionControlService.java | 175 +++++++----------- .../vc/LocalGitVersionControlService.java | 9 + .../sync/vc/GitVersionControlService.java | 5 +- 5 files changed, 95 insertions(+), 113 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 72e3d757be..38d3b00213 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -95,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixDataOrderForImport(exportDataList); + exportDataList.sort(getDataComparatorForImport()); List> importResults = new ArrayList<>(); @@ -125,9 +125,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + + @Override + public Comparator> getDataComparatorForImport() { + return Comparator.comparing(EntityExportData::getEntityType, getEntityTypeComparatorForImport()); + } + @Override - public void fixDataOrderForImport(List> exportDataList) { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 5f5bed9dac..7bdb271444 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -24,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import java.util.Comparator; import java.util.List; public interface EntitiesExportImportService { @@ -35,6 +37,9 @@ public interface EntitiesExportImportService { List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; - void fixDataOrderForImport(List> exportDataList); + + Comparator> getDataComparatorForImport(); + + Comparator getEntityTypeComparatorForImport(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index e222d027e7..f6319db9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,18 +15,10 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; @@ -34,24 +26,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -77,27 +60,16 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; - @Service @TbCoreComponent @RequiredArgsConstructor @@ -115,7 +87,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { @@ -132,23 +103,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } if (config.isAllEntities()) { - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { try { - saveEntityData(user, commit, entityId, config); + saveEntityData(user, commit, entity.getId(), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -209,22 +168,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); - result.getSaveReferencesCallback().run(); - return result; + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); } }); - try { - importResult.getSendEventsCallback().run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); - } - return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) .updated(importResult.getOldEntity() != null ? 1 : 0) .deleted(0) @@ -234,56 +191,68 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; return transactionTemplate.execute(status -> { Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); List sendEventsCallbacks = new ArrayList<>(); - // order entity types.. - // or what - versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); - AtomicInteger deleted = new AtomicInteger(); - - Set remoteEntities; - try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() - .map(VersionedEntityInfo::getExternalId) - .collect(Collectors.toSet()); - for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - if (config.isRemoveOtherEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - // need to delete entity types in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); + versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()) + .forEach(entityType -> { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + try { + int limit = 100; + int offset = 0; + List> entityDataList; + do { + entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + for (EntityExportData entityData : entityDataList) { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + } catch (Exception e) { + throw new RuntimeException(e); } + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); }); - } - results.put(entityType, VersionLoadResult.builder() - .created(created.get()) - .updated(updated.get()) - .deleted(deleted.get()) - .build()); - }); + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + VersionLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); + } + }); + }); for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { @@ -307,14 +276,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - } - @Override public List listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index b9a5a9260b..cd96f5a6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -52,6 +52,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @RequiredArgsConstructor @@ -193,6 +194,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { return gitRepositoryService.listBranches(tenantId); } + @Override + public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { + return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() + .skip(offset).limit(limit) + .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) + .collect(Collectors.toList()); + } + @Override public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { try { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index e67a8cca5f..35d68347b1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import lombok.SneakyThrows; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,7 +30,6 @@ import java.util.List; public interface GitVersionControlService { - @SneakyThrows EntitiesVersionControlSettings getSettings(TenantId tenantId); void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); @@ -57,4 +55,7 @@ public interface GitVersionControlService { List listBranches(TenantId tenantId); EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); + + List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); + } From 0b6de71768c6ff356224c177b222d5b50a1e66d1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:01:06 +0300 Subject: [PATCH 065/178] Implement version control ssh access with private key. Improve VC settings REST methods. Add tenantId field to admin settings. Add DB upgrade. --- application/pom.xml | 4 + .../main/data/upgrade/3.3.4/schema_update.sql | 3 + .../server/controller/AdminController.java | 61 +++++++- .../EntitiesVersionControlController.java | 47 +----- .../install/ThingsboardInstallService.java | 1 + .../fetch/AdminSettingsEdgeEventFetcher.java | 7 +- .../DefaultSystemDataLoaderService.java | 2 + .../install/SqlDatabaseUpgradeService.java | 12 ++ .../permission/TenantAdminPermissions.java | 1 + .../system/DefaultSystemSecurityService.java | 1 + .../DefaultEntitiesVersionControlService.java | 145 ++++++++++-------- .../vc/EntitiesVersionControlService.java | 12 +- .../server/utils/GitRepository.java | 131 +++++++++++++--- .../server/common/data/AdminSettings.java | 19 ++- .../server/common/data/StringUtils.java | 42 +++++ .../vc}/EntitiesVersionControlSettings.java | 6 +- .../data/vc/VersionControlAuthMethod.java | 21 +++ .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AdminSettingsEntity.java | 9 ++ .../server/dao/settings/AdminSettingsDao.java | 4 +- .../settings/AdminSettingsServiceImpl.java | 22 ++- .../sql/settings/AdminSettingsRepository.java | 3 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 4 +- .../main/resources/sql/schema-entities.sql | 1 + pom.xml | 5 + 25 files changed, 411 insertions(+), 154 deletions(-) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/vc}/EntitiesVersionControlSettings.java (79%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..83d36c2b42 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,6 +345,10 @@ org.eclipse.jgit org.eclipse.jgit + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index ad307a6576..f537acfb36 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -26,3 +26,6 @@ ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE customer ADD COLUMN IF NOT EXISTS external_id UUID; + +ALTER TABLE admin_settings + ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index e5934cbf26..3868476491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -20,12 +20,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; @@ -34,14 +29,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @@ -60,6 +58,9 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private EntitiesVersionControlService versionControlService; + @Autowired private UpdateService updateService; @@ -96,6 +97,7 @@ public class AdminController extends BaseController { @RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + adminSettings.setTenantId(getTenantId()); adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); @@ -180,6 +182,53 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get version control settings (getVersionControlSettings)", + notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings") + @ResponseBody + public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vsSettings") + public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + versionControlService.saveVersionControlSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", + notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) + public void checkVersionControlAccess( + @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") + @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + versionControlService.checkVersionControlAccess(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..599149402f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,25 +15,18 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; @@ -280,7 +273,7 @@ public class EntitiesVersionControlController extends BaseController { List remoteBranches = versionControlService.listBranches(getTenantId()); List infos = new ArrayList<>(); - String defaultBranch = getSettings().getDefaultBranch(); + String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); @@ -293,40 +286,6 @@ public class EntitiesVersionControlController extends BaseController { } } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @GetMapping("/settings") - public EntitiesVersionControlSettings getSettings() throws ThingsboardException { - try { - return versionControlService.getSettings(getTenantId()); - } catch (Exception e) { - throw handleException(e); - } - } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @PostMapping("/settings") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { - try { - versionControlService.saveSettings(getTenantId(), settings); - } catch (Exception e) { - throw handleException(e); - } - } - - @Data public static class BranchInfo { private final String name; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ebab403149..e728c004c0 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -219,6 +219,7 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.3.3"); case "3.3.4": log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java index 0f4ecf7e30..ee10c7859b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java @@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings))); - AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); + AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings))); @@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates))); - AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); + AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates))); @@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { } } - private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) { + private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) { AdminSettings tenantMailSettings = new AdminSettings(); + tenantMailSettings.setTenantId(tenantId); jsonValue.put("useSystemMailSettings", true); tenantMailSettings.setJsonValue(jsonValue); tenantMailSettings.setKey(key); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 9e7636879f..696ac50c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -231,6 +231,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override public void createAdminSettings() throws Exception { AdminSettings generalSettings = new AdminSettings(); + generalSettings.setTenantId(TenantId.SYS_TENANT_ID); generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); @@ -239,6 +240,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); AdminSettings mailSettings = new AdminSettings(); + mailSettings.setTenantId(TenantId.SYS_TENANT_ID); mailSettings.setKey("mail"); node = objectMapper.createObjectNode(); node.put("mailFrom", "ThingsBoard "); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index f7b9e5cad1..1f9dcf15a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -534,6 +534,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 4ff884f0e1..8f821f5051 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -28,6 +28,7 @@ public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); + put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); put(Resource.ASSET, tenantEntityPermissionChecker); put(Resource.DEVICE, tenantEntityPermissionChecker); diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 9a998544f3..2449d7fbc0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -107,6 +107,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService { AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, "securitySettings"); if (adminSettings == null) { adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); adminSettings.setKey("securitySettings"); } adminSettings.setJsonValue(JacksonUtil.valueToTree(securitySettings)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..77a5586a8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -19,34 +19,29 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.*; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -58,17 +53,11 @@ import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExp import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.*; import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; @@ -81,13 +70,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -104,7 +87,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AttributesService attributesService; + private final AdminSettingsService adminSettingsService; private final EntityService entityService; private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; @@ -114,14 +97,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; - private static final String SETTINGS_KEY = "vc"; + private static final String SETTINGS_KEY = "entitiesVersionControl"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { initRepository(tenantId, settings); @@ -412,6 +395,73 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return repository.listBranches(); } + @Override + public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + + @Override + public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); + AdminSettings adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + try { + clearRepository(tenantId); + initRepository(tenantId, savedVersionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } + return savedVersionControlSettings; + } + + @Override + public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + settings = this.restoreCredentials(settings, storedSettings); + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + try { + GitRepository.test(settings, repositoryDirectory.toFile()); + } catch (GitAPIException e) { + throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), + ThingsboardErrorCode.GENERAL); + } + } + + private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + if (storedSettings != null) { + settings.setPassword(storedSettings.getPassword()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + if (storedSettings != null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && + StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { return listVersions(tenantId, branch, null).stream() @@ -426,11 +476,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + FileUtils.forceDelete(repositoryDirectory.toFile()); Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + + GitRepository repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); } @@ -442,34 +493,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - - @SneakyThrows - @Override - public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { - attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) - )).get(); - - clearRepository(tenantId); - initRepository(tenantId, settings); - } - - @SneakyThrows - @Override - public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..2854ec1d05 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,16 +16,17 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import java.util.List; @@ -51,9 +52,12 @@ public interface EntitiesVersionControlService { List listBranches(TenantId tenantId) throws Exception; + EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + + EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; - void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..e207c30fa2 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -18,14 +18,8 @@ package org.thingsboard.server.utils; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; -import org.eclipse.jgit.api.TransportCommand; +import org.apache.sshd.common.util.security.SecurityUtils; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -34,50 +28,88 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.sshd.JGitKeyCache; +import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; +import org.eclipse.jgit.transport.sshd.SshdSessionFactory; +import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PublicKey; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; + private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; + this.sshSessionFactory = sshSessionFactory; this.directory = directory; } - public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { - CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); - Git git = Git.cloneRepository() - .setURI(uri) + public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(settings.getRepositoryUri()) .setDirectory(directory) - .setNoCheckout(true) - .setCredentialsProvider(credentialsProvider) - .call(); - return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); + .setNoCheckout(true); + configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory); + Git git = cloneCommand.call(); + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, String username, String password) throws IOException { + public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { Git git = Git.open(directory); - return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } + public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); + configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory); + lsRemoteCommand.call(); + } public void fetch() throws GitAPIException { execute(git.fetch() @@ -108,7 +140,6 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { return listCommits(branch, null, limit); } @@ -246,16 +277,66 @@ public class GitRepository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand && credentialsProvider != null) { - ((TransportCommand) command).setCredentialsProvider(credentialsProvider); + if (command instanceof TransportCommand) { + configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory); } return command.call(); } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { + if (credentialsProvider != null) { + transportCommand.setCredentialsProvider(credentialsProvider); + } + if (sshSessionFactory != null) { + transportCommand.setTransportConfigCallback(transport -> { + if (transport instanceof SshTransport) { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + } + private static CredentialsProvider newCredentialsProvider(String username, String password) { - return new UsernamePasswordCredentialsProvider(username, password); + return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password); } + private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) { + SshdSessionFactory sshSessionFactory = null; + if (StringUtils.isNotBlank(privateKey)) { + Iterable keyPairs = loadKeyPairs(privateKey, password); + sshSessionFactory = new SshdSessionFactoryBuilder() + .setPreferredAuthentications("publickey") + .setDefaultKeysProvider(file -> keyPairs) + .setHomeDirectory(directory) + .setSshDirectory(directory) + .setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() { + @Override + public List lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) { + return Collections.emptyList(); + } + + @Override + public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) { + return true; + } + }) + .build(new JGitKeyCache()); + } + return sshSessionFactory; + } + + private static Iterable loadKeyPairs(String privateKeyContent, String password) { + Iterable keyPairs = null; + try { + keyPairs = SecurityUtils.loadKeyPairIdentities(null, + null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password); + } catch (Exception e) {} + if (keyPairs == null) { + throw new IllegalArgumentException("Failed to load ssh private key"); + } + return keyPairs; + } @Data public static class Commit { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 3b639659c8..d88fabc052 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class AdminSettings extends BaseData { +public class AdminSettings extends BaseData implements HasTenantId { private static final long serialVersionUID = -7670322981725511892L; + private TenantId tenantId; + @NoXss @Length(fieldName = "key") private String key; @@ -44,6 +47,7 @@ public class AdminSettings extends BaseData { public AdminSettings(AdminSettings adminSettings) { super(adminSettings); + this.tenantId = adminSettings.getTenantId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -60,7 +64,16 @@ public class AdminSettings extends BaseData { return super.getCreatedTime(); } - @ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") + @ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true) + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + @ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") public String getKey() { return key; } @@ -69,7 +82,7 @@ public class AdminSettings extends BaseData { this.key = key; } - @ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value") + @ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value") public JsonNode getJsonValue() { return jsonValue; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index f1391555b0..52ad469ce1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -17,6 +17,10 @@ package org.thingsboard.server.common.data; public class StringUtils { + public static final String EMPTY = ""; + + public static final int INDEX_NOT_FOUND = -1; + public static boolean isEmpty(String source) { return source == null || source.isEmpty(); } @@ -32,4 +36,42 @@ public class StringUtils { public static boolean isNotBlank(String source) { return source != null && !source.isEmpty() && !source.trim().isEmpty(); } + + public static String removeStart(final String str, final String remove) { + if (isEmpty(str) || isEmpty(remove)) { + return str; + } + if (str.startsWith(remove)){ + return str.substring(remove.length()); + } + return str; + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringBetween(final String str, final String open, final String close) { + if (str == null || open == null || close == null) { + return null; + } + final int start = str.indexOf(open); + if (start != INDEX_NOT_FOUND) { + final int end = str.indexOf(close, start + open.length()); + if (end != INDEX_NOT_FOUND) { + return str.substring(start + open.length(), end); + } + } + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ce4f8e71d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java @@ -13,14 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.vc; import lombok.Data; @Data public class EntitiesVersionControlSettings { private String repositoryUri; + private VersionControlAuthMethod authMethod; private String username; private String password; + private String privateKeyFileName; + private String privateKey; + private String privateKeyPassword; private String defaultBranch; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java new file mode 100644 index 0000000000..9d2dfa048e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.vc; + +public enum VersionControlAuthMethod { + USERNAME_PASSWORD, + PRIVATE_KEY +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 0b74e79f2b..2bb9ef6bdb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -90,6 +90,8 @@ public class ModelConstants { * Cassandra admin_settings constants. */ public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; + + public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 49d5002af2..4da17d2c34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -22,14 +22,18 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; +import java.util.UUID; + import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY; @@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY @Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME) public final class AdminSettingsEntity extends BaseSqlEntity implements BaseEntity { + @Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY) + private UUID tenantId; + @Column(name = ADMIN_SETTINGS_KEY_PROPERTY) private String key; @@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl this.setUuid(adminSettings.getId().getId()); } this.setCreatedTime(adminSettings.getCreatedTime()); + this.tenantId = adminSettings.getTenantId().getId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettings toData() { AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id)); adminSettings.setCreatedTime(createdTime); + adminSettings.setTenantId(TenantId.fromUUID(tenantId)); adminSettings.setKey(key); adminSettings.setJsonValue(jsonValue); return adminSettings; diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index faea2cc34e..14455cddd4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.UUID; + public interface AdminSettingsDao extends Dao { /** @@ -35,6 +37,6 @@ public interface AdminSettingsDao extends Dao { * @param key the key * @return the admin settings object */ - AdminSettings findByKey(TenantId tenantId, String key); + AdminSettings findByTenantIdAndKey(UUID tenantId, String key); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bf4c71d296..0462cc38de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -46,20 +47,35 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) { log.trace("Executing findAdminSettingsByKey [{}]", key); Validator.validateString(key, "Incorrect key " + key); - return adminSettingsDao.findByKey(tenantId, key); + return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key); } @Override public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { + if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } + } else if (adminSettings.getKey().equals("entitiesVersionControl")) { + VersionControlAuthMethod authMethod = VersionControlAuthMethod.valueOf(adminSettings.getJsonValue().get("authMethod").asText()); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && !adminSettings.getJsonValue().has("password")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("password", vcSettings.getJsonValue().get("password").asText()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && !adminSettings.getJsonValue().has("privateKey")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKey", vcSettings.getJsonValue().get("privateKey").asText()); + if (!adminSettings.getJsonValue().has("privateKeyPassword") && vcSettings.getJsonValue().has("privateKeyPassword")) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKeyPassword", vcSettings.getJsonValue().get("privateKeyPassword").asText()); + } + } + } } - return adminSettingsDao.save(tenantId, adminSettings); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 58423645b1..10275a3c1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -25,5 +25,6 @@ import java.util.UUID; */ public interface AdminSettingsRepository extends JpaRepository { - AdminSettingsEntity findByKey(String key); + AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 438274e3cc..4a4f7e5440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -46,7 +46,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDaoorg.eclipse.jgit ${jgit.version} + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + ${jgit.version} + From 2eb94fd95e5eca1973a689802af0279124f61a7d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:38:06 +0300 Subject: [PATCH 066/178] Fix beans cycle dependency --- .../DefaultEntitiesVersionControlService.java | 2 -- .../sync/vc/EntitiesVersionControlService.java | 2 ++ .../sync/vc/LocalGitVersionControlService.java | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 1496882031..70bf5b0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -79,8 +79,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntityService entityService; private final TransactionTemplate transactionTemplate; - public static final String SETTINGS_KEY = "entitiesVersionControl"; - @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var commit = gitService.prepareCommit(user.getTenantId(), request); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 8c18aeb32d..d24fa1411e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -32,6 +32,8 @@ import java.util.List; public interface EntitiesVersionControlService { + String SETTINGS_KEY = "entitiesVersionControl"; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index aa51fc3ea1..94976438e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -38,6 +39,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -63,14 +65,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; - private final EntitiesVersionControlService entitiesVersionControlService; + private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = entitiesVersionControlService.getVersionControlSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { gitRepositoryService.initRepository(tenantId, settings); @@ -213,6 +215,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + private List listVersions(TenantId tenantId, String branch, String path) { try { return gitRepositoryService.listVersions(tenantId, branch, path); From 587066cfd32b17e6b0259f147b32987a68c94a63 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 13:19:43 +0300 Subject: [PATCH 067/178] Ability to remove version control settings --- .../server/controller/AdminController.java | 24 ++++++++++++++++--- .../DefaultEntitiesVersionControlService.java | 7 ++++++ .../vc/EntitiesVersionControlService.java | 2 ++ .../vc/LocalGitVersionControlService.java | 14 +++++++++++ .../dao/settings/AdminSettingsService.java | 2 ++ .../sync/vc/DefaultGitRepositoryService.java | 3 ++- .../service/sync/vc/GitRepositoryService.java | 2 ++ .../sync/vc/GitVersionControlService.java | 2 ++ .../server/dao/settings/AdminSettingsDao.java | 2 ++ .../settings/AdminSettingsServiceImpl.java | 6 +++++ .../sql/settings/AdminSettingsRepository.java | 4 ++++ .../dao/sql/settings/JpaAdminSettingsDao.java | 9 +++++++ 12 files changed, 73 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 237f681819..bf8c656257 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; @@ -38,8 +41,8 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; -import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -203,7 +206,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vsSettings") + @PostMapping("/vcSettings") public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -213,6 +216,21 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + versionControlService.deleteVersionControlSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 70bf5b0d5b..c69ff94630 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -312,6 +312,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return savedVersionControlSettings; } + @Override + public void deleteVersionControlSettings(TenantId tenantId) { + if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + gitService.clearRepository(tenantId); + } + } + @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index d24fa1411e..28f095123c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -58,6 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + void deleteVersionControlSettings(TenantId tenantId); + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index 94976438e9..d73157d256 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -111,6 +111,20 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + @Override + public void clearRepository(TenantId tenantId) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.clearRepository(tenantId); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + @Override public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { var lock = getRepoLock(tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index d9238c9d7b..77f542ccc3 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -27,4 +27,6 @@ public interface AdminSettingsService { AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); + boolean deleteAdminSettings(TenantId tenantId, String key); + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 56ca38ffe4..aac40e0dcb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -224,7 +224,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { repositories.put(tenantId, repository); } - private void clearRepository(TenantId tenantId) throws IOException { + @Override + public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { FileUtils.deleteDirectory(new File(repository.getDirectory())); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 4b1843e3c7..19754750c2 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -36,6 +36,8 @@ public interface GitRepositoryService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void clearRepository(TenantId tenantId) throws IOException; + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index 8345375e4f..f28584c418 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -34,6 +34,8 @@ public interface GitVersionControlService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + void clearRepository(TenantId tenantId); + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); void addToCommit(PendingCommit commit, EntityExportData> entityData); diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index 14455cddd4..d211a8340f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -39,4 +39,6 @@ public interface AdminSettingsDao extends Dao { */ AdminSettings findByTenantIdAndKey(UUID tenantId, String key); + boolean removeByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 0bb5564fa6..433d056186 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -62,4 +62,10 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { return adminSettingsDao.save(tenantId, adminSettings); } + @Override + public boolean deleteAdminSettings(TenantId tenantId, String key) { + log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); + Validator.validateString(key, "Incorrect key " + key); + return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 10275a3c1a..0e68327fdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -27,4 +27,8 @@ public interface AdminSettingsRepository extends JpaRepository Date: Thu, 19 May 2022 13:21:19 +0300 Subject: [PATCH 068/178] Clear repository on settings update --- .../service/sync/vc/DefaultEntitiesVersionControlService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c69ff94630..dcc96b7d17 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -305,6 +305,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException("Failed to load version control settings!", e); } try { + gitService.clearRepository(tenantId); gitService.initRepository(tenantId, savedVersionControlSettings); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); From 10248aa9f64c5d90d2692cf35f7b803b0f11d384 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 19 May 2022 17:49:40 +0300 Subject: [PATCH 069/178] fix of the packge list --- .../server/vc/ThingsboardVersionControlExecutorApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index 2724a566ee..d30818110d 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -25,7 +25,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; From 76af741399fc01f2eb2b735802f00eb2e932f755 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 15:02:30 +0300 Subject: [PATCH 070/178] UI: Implement git settings form --- .../server/controller/AdminController.java | 8 +- .../DefaultEntitiesVersionControlService.java | 21 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 2 + ui-ngx/src/app/core/http/admin.service.ts | 20 ++ ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../version-control-settings.component.html | 110 ++++++++++ .../version-control-settings.component.scss | 33 +++ .../version-control-settings.component.ts | 198 ++++++++++++++++++ ui-ngx/src/app/shared/models/constants.ts | 3 +- .../src/app/shared/models/settings.models.ts | 21 ++ .../assets/locale/locale.constant-en_US.json | 24 ++- 13 files changed, 459 insertions(+), 13 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index bf8c656257..45b8aa5f95 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -207,10 +207,14 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public EntitiesVersionControlSettings saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - versionControlService.saveVersionControlSettings(getTenantId(), settings); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.saveVersionControlSettings(getTenantId(), settings)); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..d46884ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -291,11 +291,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + EntitiesVersionControlSettings storedSettings = null; + if (adminSettings != null) { + try { + storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - AdminSettings adminSettings = new AdminSettings(); - adminSettings.setTenantId(tenantId); - adminSettings.setKey(SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -341,8 +351,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { if (storedSettings != null) { settings.setPrivateKey(storedSettings.getPrivateKey()); - if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && - StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 7257cfb57b..de4a2fd5ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -51,6 +52,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDao { + return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings', versionControlSettings, + defaultHttpOptionsFromConfig(config)); + } + + public deleteEntitiesVersionControlSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + } + + public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + } + public checkUpdates(config?: RequestConfig): Observable { return this.http.get(`/api/admin/updates`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 4a52fc66f6..85cf24e04d 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -350,7 +350,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '80px', + height: '120px', icon: 'settings', pages: [ { @@ -366,6 +366,13 @@ export class MenuService { type: 'link', path: '/settings/resources-library', icon: 'folder' + }, + { + id: guid(), + name: 'admin.git-settings', + type: 'link', + path: '/settings/vc', + icon: 'manage_history' } ] } @@ -500,6 +507,11 @@ export class MenuService { name: 'resource.resources-library', icon: 'folder', path: '/settings/resources-library' + }, + { + name: 'admin.git-settings', + icon: 'manage_history', + path: '/settings/vc', } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index ddccbb4da4..6c4963d06e 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -32,6 +32,7 @@ import { ResourcesLibraryTableConfigResolver } from '@home/pages/admin/resource/ import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -183,6 +184,19 @@ const routes: Routes = [ } } ] + }, + { + path: 'vc', + component: VersionControlSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.git-settings', + breadcrumb: { + label: 'admin.git-settings', + icon: 'manage_history' + } + } } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 326b16f950..cb81e56020 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -28,6 +28,7 @@ import { SmsProviderComponent } from '@home/pages/admin/sms-provider.component'; import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dialog.component'; import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @NgModule({ declarations: @@ -39,7 +40,8 @@ import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources- SecuritySettingsComponent, OAuth2SettingsComponent, HomeSettingsComponent, - ResourcesLibraryComponent + ResourcesLibraryComponent, + VersionControlSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html new file mode 100644 index 0000000000..b512bb96f5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html @@ -0,0 +1,110 @@ + +
+ + +
+ admin.git-repository-settings + +
+
+ + + +
+ +
+
+ + admin.repository-url + + + admin.repository-url-required + + + + admin.default-branch + + +
+ admin.authentication-settings + + admin.auth-method + + + {{versionControlAuthMethodTranslations.get(method) | translate}} + + + +
+ + common.username + + + + {{ 'admin.change-password-access-token' | translate }} + + + admin.password-access-token + + + +
+
+ + + + {{ 'admin.change-passphrase' | translate }} + + + admin.passphrase + + + +
+
+
+ + + + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss new file mode 100644 index 0000000000..ede3570e68 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts new file mode 100644 index 0000000000..c2626ec062 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts @@ -0,0 +1,198 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + EntitiesVersionControlSettings, + VersionControlAuthMethod, + versionControlAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; + +@Component({ + selector: 'tb-version-control-settings', + templateUrl: './version-control-settings.component.html', + styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] +}) +export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + versionControlSettingsForm: FormGroup; + settings: EntitiesVersionControlSettings = null; + + versionControlAuthMethod = VersionControlAuthMethod; + versionControlAuthMethods = Object.values(VersionControlAuthMethod); + versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.versionControlSettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: [null, []], + authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + (settings) => { + this.settings = settings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + }); + } + + checkAccess(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.checkVersionControlAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-git-settings-title', ), + this.translate.instant('admin.delete-git-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteEntitiesVersionControlSettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + } + ); + } + }); + } + + confirmForm(): FormGroup { + return this.versionControlSettingsForm; + } + + changePasswordChanged() { + if (this.changePassword) { + this.versionControlSettingsForm.get('password').patchValue(''); + this.versionControlSettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); + this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; + if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.versionControlSettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.versionControlSettingsForm.get('password').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('password').disable({emitEvent}); + } + this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.versionControlSettingsForm.get('username').disable({emitEvent}); + this.versionControlSettingsForm.get('password').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.versionControlSettingsForm.get('privateKey').clearValidators(); + } else { + this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 4bfce6414a..7c970678b8 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -133,7 +133,8 @@ export const HelpLinks = { widgetsConfigAlarm: helpBaseUrl + '/docs/user-guide/ui/dashboards#alarm', widgetsConfigStatic: helpBaseUrl + '/docs/user-guide/ui/dashboards#static', ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', - ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge' + ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', + versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 615d0994c9..8a1b316001 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -396,3 +396,24 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid } return smsProviderConfiguration; } + +export enum VersionControlAuthMethod { + USERNAME_PASSWORD = 'USERNAME_PASSWORD', + PRIVATE_KEY = 'PRIVATE_KEY' +} + +export const versionControlAuthMethodTranslationMap = new Map([ + [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +]); + +export interface EntitiesVersionControlSettings { + repositoryUri: string; + defaultBranch: string; + authMethod: VersionControlAuthMethod; + username: string; + password: string; + privateKeyFileName: string; + privateKey: string; + privateKeyPassword: string; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cf576aedbf..5aaadb2bd6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -311,8 +311,28 @@ "scheme-music-codes": "10 - Music Codes (ISO-2022-JP)", "scheme-extended-kanji-jis": "13 - Extended Kanji JIS (X 0212-1990)", "scheme-korean-graphic-character-set": "14 - Korean Graphic Character Set (KS C 5601/KS X 1001)" - } - }, + }, + "git-settings": "Git settings", + "git-repository-settings": "Git repository settings", + "repository-url": "Repository URL", + "repository-url-required": "Repository URL is required.", + "default-branch": "Default branch name", + "authentication-settings": "Authentication settings", + "auth-method": "Authentication method", + "auth-method-username-password": "Password / access token", + "auth-method-private-key": "Private key", + "password-access-token": "Password / access token", + "change-password-access-token": "Change password / access token", + "private-key": "Private key", + "drop-private-key-file-or": "Drag and drop a private key file or", + "passphrase": "Passphrase", + "enter-passphrase": "Enter passphrase", + "change-passphrase": "Change passphrase", + "check-access": "Check access", + "check-vc-access-success": "Git repository access successfully verified!", + "delete-git-settings-title": "Are you sure you want to delete git settings?", + "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + }, "alarm": { "alarm": "Alarm", "alarms": "Alarms", From 7071c7a6bc9deb72bf579e4db868c7d98146c529 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 20 May 2022 17:33:24 +0300 Subject: [PATCH 071/178] Implementation of the Git Queue service client --- .../server/controller/BaseController.java | 21 + .../EntitiesVersionControlController.java | 85 ++-- .../install/SqlDatabaseUpgradeService.java | 12 - .../queue/DefaultTbClusterService.java | 8 + .../service/sync/vc/CommitGitRequest.java | 33 ++ .../DefaultEntitiesVersionControlService.java | 215 +++++---- .../DefaultGitVersionControlQueueService.java | 287 +++++++++++ .../sync/vc/EntitiesContentGitRequest.java | 37 ++ .../vc/EntitiesVersionControlService.java | 19 +- .../sync/vc/EntityContentGitRequest.java | 34 ++ .../vc/GitVersionControlQueueService.java | 57 +++ .../sync/vc/ListBranchesGitRequest.java | 29 ++ .../sync/vc/ListEntitiesGitRequest.java | 30 ++ .../sync/vc/ListVersionsGitRequest.java | 31 ++ .../vc/LocalGitVersionControlService.java | 452 +++++++++--------- .../service/sync/vc/PendingGitRequest.java | 37 ++ .../src/main/resources/thingsboard.yml | 5 +- .../server/cluster/TbClusterService.java | 7 +- common/cluster-api/src/main/proto/queue.proto | 96 +++- .../server/common/msg/queue/ServiceType.java | 2 +- .../queue/discovery/HashPartitionService.java | 10 +- .../queue/kafka/TbKafkaTopicConfigs.java | 6 + .../provider/AwsSqsMonolithQueueFactory.java | 7 + .../provider/AwsSqsTbCoreQueueFactory.java | 7 + .../InMemoryMonolithQueueFactory.java | 6 + .../provider/KafkaMonolithQueueFactory.java | 36 +- .../provider/KafkaTbCoreQueueFactory.java | 20 + .../KafkaTbVersionControlQueueFactory.java | 116 +++++ .../provider/PubSubMonolithQueueFactory.java | 7 + .../provider/PubSubTbCoreQueueFactory.java | 7 + .../RabbitMqMonolithQueueFactory.java | 7 + .../provider/RabbitMqTbCoreQueueFactory.java | 7 + .../ServiceBusMonolithQueueFactory.java | 7 + .../ServiceBusTbCoreQueueFactory.java | 7 + .../queue/provider/TbCoreQueueFactory.java | 8 + .../provider/TbCoreQueueProducerProvider.java | 8 + .../provider/TbQueueProducerProvider.java | 8 + .../TbRuleEngineProducerProvider.java | 6 + .../TbTransportQueueProducerProvider.java | 6 + .../TbVersionControlProducerProvider.java | 84 ++++ .../TbVersionControlQueueFactory.java | 44 ++ .../TbQueueVersionControlSettings.java | 34 ++ .../queue/util/TbVersionControlComponent.java | 26 + common/version-control/pom.xml | 24 - .../sync/vc/GitVersionControlService.java | 11 - .../version-control/src/main/proto/vc.proto | 71 --- 46 files changed, 1579 insertions(+), 498 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java delete mode 100644 common/version-control/src/main/proto/vc.proto diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 195ea9a3ee..709b809823 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -18,6 +18,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -27,6 +31,7 @@ import org.springframework.http.MediaType; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -948,4 +953,20 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + protected DeferredResult wrapFuture(ListenableFuture future) { + final DeferredResult deferredResult = new DeferredResult<>(); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(T result) { + deferredResult.setResult(result); + } + + @Override + public void onFailure(Throwable t) { + deferredResult.setErrorResult(t); + } + }, MoreExecutors.directExecutor()); + return deferredResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..4c5c7dad65 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,16 +15,26 @@ */ package org.thingsboard.server.controller; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.claim.ClaimResponse; +import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -34,6 +44,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -90,16 +101,15 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/version") - public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - return versionControlService.saveEntitiesVersion(user, request); + return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } } - @ApiOperation(value = "", notes = "" + "```\n[\n" + " {\n" + @@ -108,12 +118,12 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + public DeferredResult> listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId)); } catch (Exception e) { throw handleException(e); } @@ -127,10 +137,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + public DeferredResult> listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType)); } catch (Exception e) { throw handleException(e); } @@ -152,9 +162,9 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + public DeferredResult> listVersions(@PathVariable String branch) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + return wrapFuture(versionControlService.listVersions(getTenantId(), branch)); } catch (Exception e) { throw handleException(e); } @@ -162,21 +172,21 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping("/entity/{branch}/{entityType}/{versionId}") - public List listEntitiesAtVersion(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); + return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } catch (Exception e) { throw handleException(e); } } @GetMapping("/entity/{branch}/{versionId}") - public List listAllEntitiesAtVersion(@PathVariable String branch, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } catch (Exception e) { throw handleException(e); } @@ -216,20 +226,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - String versionId = request.getVersionId(); - if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); - } else { - throw new IllegalArgumentException("No versions available in branch"); - } - } - - return versionControlService.loadEntitiesVersion(user, request); + return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } @@ -252,19 +252,22 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n\n```") @GetMapping("/branches") - public List listBranches() throws ThingsboardException { + public DeferredResult> listBranches() throws ThingsboardException { try { - List remoteBranches = versionControlService.listBranches(getTenantId()); - List infos = new ArrayList<>(); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); - String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); - infos.add(new BranchInfo(defaultBranch, true)); - } + String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); - return infos; + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + }, MoreExecutors.directExecutor())); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b1bdbf61b8..db2352b706 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -647,18 +647,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.3.4": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); - loadSql(schemaUpdateFile, conn); - log.info("Updating schema settings..."); - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); - log.info("Schema updated."); - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2806c35cda..4a52ea2287 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -135,6 +135,14 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java new file mode 100644 index 0000000000..b510ffa62c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +public class CommitGitRequest extends PendingGitRequest { + + private final VersionCreateRequest request; + + public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { + super(tenantId); + this.request = request; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..3fb47a8bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,12 +15,18 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,7 +39,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.*; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -56,6 +61,8 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -72,118 +79,137 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final GitVersionControlService gitService; + private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; + private ListeningExecutorService executor; + + @Value("${vc.thread_pool_size:4}") + private int threadPoolSize; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class)); + } + + @PreDestroy + public void shutdown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + @SuppressWarnings("UnstableApiUsage") @Override - public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); + public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - switch (request.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); - break; - } - case COMPLEX: { - ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitService.deleteAll(commit, entityType); - } + return Futures.transformAsync(pendingCommit, commit -> { + List> gitFutures = new ArrayList<>(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + break; + } + case COMPLEX: { + ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { + gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + } - if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - try { - saveEntityData(user, commit, entity.getId(), config); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } else { - for (UUID entityId : config.getEntityIds()) { - try { - saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } } } - } - - }); - break; + }); + break; + } } - } - - return gitService.push(commit); + return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + }, executor); } - private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { + private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - gitService.addToCommit(commit, entityData); + return gitServiceQueue.addToCommit(commit, entityData); } - @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, externalId); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, entityType); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch); } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override - public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId); + public ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId); } + @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); + ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + Futures.transform(future, entityData -> { + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + }, executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return transactionTemplate.execute(status -> { + return executor.submit(() -> transactionTemplate.execute(status -> { Map results = new HashMap<>(); Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); @@ -199,9 +225,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { int limit = 100; int offset = 0; - List> entityDataList; + List entityDataList; do { - entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) @@ -215,7 +241,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } offset += limit; importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); } while (entityDataList.size() == limit); } catch (Exception e) { throw new RuntimeException(e); @@ -263,7 +289,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } return new ArrayList<>(results.values()); - }); + })); } default: throw new IllegalArgumentException("Unsupported version load request"); @@ -272,8 +298,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listBranches(TenantId tenantId) throws Exception { - return gitService.listBranches(tenantId); + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + return gitServiceQueue.listBranches(tenantId); } @Override @@ -304,32 +330,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } + //TODO: ashvayka +// try { +// gitService.clearRepository(tenantId); +// gitService.initRepository(tenantId, savedVersionControlSettings); +// } catch (Exception e) { +// throw new RuntimeException("Failed to init repository!", e); +// } return savedVersionControlSettings; } @Override public void deleteVersionControlSettings(TenantId tenantId) { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { - gitService.clearRepository(tenantId); - } + //TODO: ashvayka +// if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { +// gitService.clearRepository(tenantId); +// } } @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); settings = this.restoreCredentials(settings, storedSettings); - try { - gitService.testRepository(tenantId, settings); - } catch (Exception e) { - throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), - ThingsboardErrorCode.GENERAL); - } + //TODO: ashvayka +// try { +// gitService.testRepository(tenantId, settings); +// } catch (Exception e) { +// throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), +// ThingsboardErrorCode.GENERAL); +// } } private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java new file mode 100644 index 0000000000..410847fb6b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -0,0 +1,287 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +@TbCoreComponent +@Service +@RequiredArgsConstructor +public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final TbServiceInfoProvider serviceInfoProvider; + private final TbClusterService clusterService; + private final Map> pendingRequestMap = new HashMap<>(); + + @Override + public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + SettableFuture future = SettableFuture.create(); + + CommitGitRequest commit = new CommitGitRequest(tenantId, request); + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPrepareMsg(getCommitPrepareMsg(request)).build() + ).build(), wrap(future, commit)); + + return future; + } + + @Override + public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJson(entityDataJson).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + return future; + } + + @Override + public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityType, null); + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setDeleteMsg( + TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + + return future; + } + + @Override + public ListenableFuture push(CommitGitRequest commit) { + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPushMsg( + TransportProtos.PushMsg.newBuilder().build() + ).build() + ).build(), wrap(commit.getFuture())); + + return commit.getFuture(); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .build()); + } + + private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { + ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .build()); + } + + private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { + ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listBranches(TenantId tenantId) { + ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + + @Override + public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { + EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() + , wrap(request.getFuture())); + + return request.getFuture(); +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } + } + + @Override + public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { + EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); + + registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityType.name()) + .setOffset(offset) + .setLimit(limit) + ).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + private static TbQueueCallback wrap(SettableFuture future) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static TbQueueCallback wrap(SettableFuture future, T value) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + future.set(value); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static String getRelativePath(EntityType entityType, EntityId entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + } + + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request) { + var tenantId = request.getTenantId(); + var requestId = request.getRequestId(); + return ToVersionControlServiceMsg.newBuilder() + .setNodeId(serviceInfoProvider.getServiceId()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setRequestIdMSB(requestId.getMostSignificantBits()) + .setRequestIdLSB(requestId.getLeastSignificantBits()); + + } +} + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java new file mode 100644 index 0000000000..2ded932e44 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +import java.util.List; + +@Getter +public class EntitiesContentGitRequest extends PendingGitRequest> { + + private final String versionId; + private final EntityType entityType; + + public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { + super(tenantId); + this.versionId = versionId; + this.entityType = entityType; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..02afafedad 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -34,25 +35,23 @@ public interface EntitiesVersionControlService { String SETTINGS_KEY = "entitiesVersionControl"; - VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; + ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; - List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - - - List listBranches(TenantId tenantId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java new file mode 100644 index 0000000000..c9ce3c76af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Getter +public class EntityContentGitRequest extends PendingGitRequest { + + private final String versionId; + private final EntityId entityId; + + public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { + super(tenantId); + this.versionId = versionId; + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java new file mode 100644 index 0000000000..e7b7171ea0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlQueueService { + + ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + + ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); + + ListenableFuture deleteAll(CommitGitRequest pendingCommit, EntityType entityType); + + ListenableFuture push(CommitGitRequest commit); + + ListenableFuture> listVersions(TenantId tenantId, String branch); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + ListenableFuture> listBranches(TenantId tenantId); + + ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); + + ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java new file mode 100644 index 0000000000..c8219641da --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListBranchesGitRequest extends PendingGitRequest> { + + public ListBranchesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java new file mode 100644 index 0000000000..0c7216ef49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListEntitiesGitRequest extends PendingGitRequest> { + + public ListEntitiesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java new file mode 100644 index 0000000000..0c8e51042b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public class ListVersionsGitRequest extends PendingGitRequest> { + + public ListVersionsGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..66803e062d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -19,18 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +35,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -60,235 +56,235 @@ import java.util.stream.Collectors; @RequiredArgsConstructor @Service @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService implements GitVersionControlService { +public class LocalGitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); - if (settings != null) { - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - } - - @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.testRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void clearRepository(TenantId tenantId) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.clearRepository(tenantId); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - var pendingCommit = new PendingCommit(tenantId, request); - PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); - if (old != null) { - gitRepositoryService.abort(old); - } - gitRepositoryService.prepareCommit(pendingCommit); - return pendingCommit; - } finally { - lock.unlock(); - } - } - - @Override - public void deleteAll(PendingCommit commit, EntityType entityType) { - doInsideLock(commit, c -> { - try { - gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public void addToCommit(PendingCommit commit, EntityExportData> entityData) { - doInsideLock(commit, c -> { - String entityDataJson; - try { - entityDataJson = jsonWriter.writeValueAsString(entityData); - gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString()), entityDataJson); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); - } - - @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { - try { - return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { - return listEntitiesAtVersion(tenantId, branch, versionId, null); - } - - @Override - public List listBranches(TenantId tenantId) { - return gitRepositoryService.listBranches(tenantId); - } - - @Override - public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { - return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() - .skip(offset).limit(limit) - .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) - .collect(Collectors.toList()); - } - - @Override - public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { - try { - String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, - getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - } - - private List listVersions(TenantId tenantId, String branch, String path) { - try { - return gitRepositoryService.listVersions(tenantId, branch, path); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private void doInsideLock(PendingCommit commit, Consumer r) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - r.accept(commit); - } finally { - lock.unlock(); - } - } - - private T executeInsideLock(PendingCommit commit, Function c) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - return c.apply(commit); - } finally { - lock.unlock(); - } - } - - private void checkCommit(PendingCommit commit) { - PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); - if (existing == null || !existing.getTxId().equals(commit.getTxId())) { - throw new ConcurrentModificationException(); - } - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; - } - - private Lock getRepoLock(TenantId tenantId) { - return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); - } +// private final Map pendingCommitMap = new HashMap<>(); +// +// @AfterStartUp +// public void init() { +// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { +// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); +// if (settings != null) { +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// log.warn("Failed to init repository for tenant {}", tenantId, e); +// } +// } +// }); +// } +// +// @Override +// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.testRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void clearRepository(TenantId tenantId) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.clearRepository(tenantId); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// var pendingCommit = new PendingCommit(tenantId, request); +// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); +// if (old != null) { +// gitRepositoryService.abort(old); +// } +// gitRepositoryService.prepareCommit(pendingCommit); +// return pendingCommit; +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void deleteAll(PendingCommit commit, EntityType entityType) { +// doInsideLock(commit, c -> { +// try { +// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { +// doInsideLock(commit, c -> { +// String entityDataJson; +// try { +// entityDataJson = jsonWriter.writeValueAsString(entityData); +// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), +// entityData.getEntity().getId().toString()), entityDataJson); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public VersionCreationResult push(PendingCommit commit) { +// return executeInsideLock(commit, gitRepositoryService::push); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch) { +// return listVersions(tenantId, branch, (String) null); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { +// return listVersions(tenantId, branch, getRelativePath(entityType, null)); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { +// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { +// try { +// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { +// return listEntitiesAtVersion(tenantId, branch, versionId, null); +// } +// +// @Override +// public List listBranches(TenantId tenantId) { +// return gitRepositoryService.listBranches(tenantId); +// } +// +// @Override +// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { +// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() +// .skip(offset).limit(limit) +// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) +// .collect(Collectors.toList()); +// } +// +// @Override +// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { +// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); +// if (adminSettings != null) { +// try { +// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); +// } catch (Exception e) { +// throw new RuntimeException("Failed to load version control settings!", e); +// } +// } +// return null; +// } +// +// private List listVersions(TenantId tenantId, String branch, String path) { +// try { +// return gitRepositoryService.listVersions(tenantId, branch, path); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private void doInsideLock(PendingCommit commit, Consumer r) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// r.accept(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private T executeInsideLock(PendingCommit commit, Function c) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// return c.apply(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private void checkCommit(PendingCommit commit) { +// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); +// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { +// throw new ConcurrentModificationException(); +// } +// } +// +// private String getRelativePath(EntityType entityType, String entityId) { +// String path = entityType.name().toLowerCase(); +// if (entityId != null) { +// path += "/" + entityId + ".json"; +// } +// return path; +// } +// +// private Lock getRepoLock(TenantId tenantId) { +// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java new file mode 100644 index 0000000000..3650f9e113 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.SettableFuture; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Getter +public class PendingGitRequest { + + private final UUID requestId; + private final TenantId tenantId; + private final SettableFuture future; + + public PendingGitRequest(TenantId tenantId) { + this.requestId = UUID.randomUUID(); + this.tenantId = tenantId; + this.future = SettableFuture.create(); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 650f110dd7..1032c79054 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1021,6 +1021,9 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1115,8 +1118,8 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: + thread_pool_size: "${TB_VC_POOL_SIZE:4}" git: - service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" management: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 22953f259a..e2486f3a7e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -31,8 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueClusterService; @@ -47,9 +48,11 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 50f114856f..cb1a1ab4db 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -618,8 +618,8 @@ message TbSubscriptionUpdateValueListProto { } message TbSubscriptionUpdateTsValue { - int64 ts = 1; - optional string value = 2; + int64 ts = 1; + optional string value = 2; } /** @@ -676,6 +676,95 @@ message EdgeNotificationMsgProto { PostAttributeMsg postAttributesMsg = 12; } +/** + TB Core to Version Control Service + */ +message CommitRequestMsg { + PrepareMsg prepareMsg = 1; + AddMsg addMsg = 2; + DeleteMsg deleteMsg = 3; + PushMsg pushMsg = 4; + AbortMsg abortMsg = 5; +} + +message CommitResponseMsg { + string commitId = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string commitMsg = 1; + string branchName = 2; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} + +message ListVersionsRequestMsg { + string branchName = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message ListEntitiesRequestMsg { + string branchName = 1; + string versionId = 2; + string entityType = 3; +} + +message ListBranchesRequestMsg { +} + +message EntityContentRequestMsg { + string versionId = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message EntitiesContentRequestMsg { + string versionId = 1; + string entityType = 2; + int32 offset = 3; + int32 limit = 4; +} + +message ToVersionControlServiceMsg { + string nodeId = 1; + int64 tenantIdMSB = 2; + int64 tenantIdLSB = 3; + int64 requestIdMSB = 4; + int64 requestIdLSB = 5; + CommitRequestMsg commitRequest = 6; + ListVersionsRequestMsg listVersionRequest = 7; + ListEntitiesRequestMsg listEntitiesRequest = 8; + ListBranchesRequestMsg listBranchesRequest = 9; + EntityContentRequestMsg entityContentRequest = 10; + EntitiesContentRequestMsg entitiesContentRequest = 11; +} + +message VersionControlResponseMsg { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + CommitResponseMsg commitResponse = 3; +} + /** * Main messages; */ @@ -730,6 +819,7 @@ message ToCoreNotificationMsg { bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; + VersionControlResponseMsg vcResponseMsg = 7; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -793,3 +883,5 @@ message ToOtaPackageStateServiceMsg { int64 otaPackageIdLSB = 7; string type = 8; } + + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 1c05ac7f66..7f276e7e3f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR, TB_VC_EXECUTOR; public static ServiceType of(String serviceType) { return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index ccf8b9dd92..e2325be878 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,6 +54,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; + @Value("${queue.vc.topic}") + private String vcTopic; + @Value("${queue.vc.partitions:10}") + private Integer vcPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -97,6 +101,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); + partitionSizesMap.put(vcKey, vcPartitions); + partitionTopicsMap.put(vcKey, vcTopic); + List queueRoutingInfoList; String serviceType = serviceInfoProvider.getServiceType(); @@ -401,7 +409,7 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); - } else if (ServiceType.TB_CORE.equals(serviceType)) { + } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 4c9194b675..75d0bd9b2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -40,6 +40,9 @@ public class TbKafkaTopicConfigs { private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; + @Value("${queue.kafka.topic-properties.version-control:}") + private String vcProperties; + @Getter private Map coreConfigs; @@ -53,6 +56,8 @@ public class TbKafkaTopicConfigs { private Map jsExecutorConfigs; @Getter private Map fwUpdatesConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -62,6 +67,7 @@ public class TbKafkaTopicConfigs { notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); fwUpdatesConfigs = getConfigs(fwUpdatesProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 660173c71d..d02b58aa2b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 81736c2b55..89b45e2822 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fe87fc2886..98c1a8029e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -146,6 +146,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 104c531e12..b24ae700bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,6 +52,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbKafkaSettings kafkaSettings; @@ -68,6 +70,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -76,6 +79,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -85,6 +90,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.notificationsTopicService = notificationsTopicService; @@ -95,6 +101,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -103,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -155,6 +163,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("monolith-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { String queueName = configuration.getName(); @@ -311,6 +332,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -331,5 +362,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 58dec0792a..f476a068d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -50,6 +51,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -65,6 +67,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; @@ -74,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; public KafkaTbCoreQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -82,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { @@ -92,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; @@ -101,6 +107,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -282,6 +289,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -302,5 +319,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..598f86d0f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") +public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + this.consumerStatsService = consumerStatsService; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index aceb9d4f0d..a57e30b655 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -207,6 +208,12 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 18a6668581..6cf9aa9a45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 2a724289bb..d4839110e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e728be6085..1dca3ba551 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -169,6 +170,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 88bb0a4045..f7db6379c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -204,6 +205,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e1eb41b2b2..5b471b2830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 4debe1e280..aadefa0292 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateSer import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -122,4 +123,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueProducer> createTransportApiResponseProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + /** + * Used to push messages to instances of TB Version Control Service + * + * @return + */ + TbQueueProducer> createVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index ef0b0c38e9..ed7de35274 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -22,6 +22,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -39,6 +40,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toVersionControl; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -52,6 +54,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); } @Override @@ -83,4 +86,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + return toVersionControl; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 19ebb4666e..19046c5a2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -70,4 +71,11 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbUsageStatsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 26f9df069e..c21ae99b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -83,4 +84,9 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3132ed684f..f9cc139559 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -73,6 +74,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java new file mode 100644 index 0000000000..c8ffc1da0d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") +public class TbVersionControlProducerProvider implements TbQueueProducerProvider { + + private final TbVersionControlQueueFactory tbQueueProvider; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; + + public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java new file mode 100644 index 0000000000..70b16afc68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages from TB Core Service + * + * @return + */ + TbQueueConsumer> createToVersionControlMsgConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java new file mode 100644 index 0000000000..e7f0ce2259 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueVersionControlSettings { + + @Value("${queue.vc.topic:tb_version_control}") + private String topic; + + @Value("${queue.vc.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + + @Value("${queue.vc.partitions:10}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java new file mode 100644 index 0000000000..e72fe60b6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") +public @interface TbVersionControlComponent { +} diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index a8d775bcdc..7b46f7ba72 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,21 +86,6 @@ org.eclipse.jgit org.eclipse.jgit - - io.grpc - grpc-netty-shaded - provided - - - io.grpc - grpc-protobuf - provided - - - io.grpc - grpc-stub - provided - org.eclipse.jgit org.eclipse.jgit.ssh.apache @@ -126,15 +111,6 @@ - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - thingsboard-repo-deploy diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..56349cf8a5 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -16,15 +16,12 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; @@ -36,14 +33,6 @@ public interface GitVersionControlService { void clearRepository(TenantId tenantId); - PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); - - void addToCommit(PendingCommit commit, EntityExportData> entityData); - - void deleteAll(PendingCommit pendingCommit, EntityType entityType); - - VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); List listVersions(TenantId tenantId, String branch, EntityType entityType); diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto deleted file mode 100644 index e8c7e09384..0000000000 --- a/common/version-control/src/main/proto/vc.proto +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "org.thingsboard.server.gen.vc.v1"; -option java_multiple_files = true; -option java_outer_classname = "EdgeProtos"; - -package vc; - -// Interface exported by the ThingsBoard Core. -service TbGitRpcService { - - rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} - -} - - -/** - * Data Structures; - */ -message CommitRequestMsg { - string txId = 1; - PrepareMsg prepareMsg = 2; - AddMsg addMsg = 3; - DeleteMsg deleteMsg = 4; - PushMsg pushMsg = 5; - AbortMsg abortMsg = 6; -} - -message CommitResponseMsg { - string id = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; -} - -message PrepareMsg { - string tenantId = 1; - string commitMsg = 2; - string branchName = 3; -} - -message AddMsg { - string relativePath = 1; - string entityDataJson = 2; -} - -message DeleteMsg { - string relativePath = 1; -} - -message PushMsg { -} - -message AbortMsg { -} \ No newline at end of file From 841f9d5ff6287af4f6ba3e39b7bff3d09a76ab70 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 17:50:35 +0300 Subject: [PATCH 072/178] UI: Single entity Git export --- .../http/entities-version-control.service.ts | 40 ++++ .../home/components/home-components.module.ts | 7 +- .../profile/device-profile.component.html | 6 + .../vc/vc-entity-export-dialog.component.html | 77 +++++++ .../vc/vc-entity-export-dialog.component.ts | 105 +++++++++ .../home/dialogs/home-dialogs.service.ts | 17 ++ .../home/pages/asset/asset.component.html | 6 + .../asset/assets-table-config.resolver.ts | 10 + .../pages/customer/customer.component.html | 6 + .../customers-table-config.resolver.ts | 12 ++ .../dashboard/dashboard-form.component.html | 6 + .../dashboards-table-config.resolver.ts | 12 ++ .../device-profiles-table-config.resolver.ts | 12 ++ .../home/pages/device/device.component.html | 6 + .../device/devices-table-config.resolver.ts | 10 + .../pages/rulechain/rulechain.component.html | 6 + .../rulechains-table-config.resolver.ts | 12 ++ .../vc/branch-autocomplete.component.html | 43 ++++ .../vc/branch-autocomplete.component.ts | 199 ++++++++++++++++++ ui-ngx/src/app/shared/models/vc.models.ts | 55 +++++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 12 ++ 22 files changed, 662 insertions(+), 4 deletions(-) create mode 100644 ui-ngx/src/app/core/http/entities-version-control.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/vc.models.ts diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts new file mode 100644 index 0000000000..231581a02f --- /dev/null +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -0,0 +1,40 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable } from 'rxjs'; +import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; + +@Injectable({ + providedIn: 'root' +}) +export class EntitiesVersionControlService { + + constructor( + private http: HttpClient + ) { + } + + public listBranches(config?: RequestConfig): Observable> { + return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 4a721647ef..a88f0349eb 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -151,6 +151,7 @@ import { DashboardStateComponent } from '@home/components/dashboard-page/dashboa import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; +import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; @NgModule({ declarations: @@ -272,7 +273,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], imports: [ CommonModule, @@ -388,7 +390,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html index 565eb5b160..7e35dbdaf4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html @@ -34,6 +34,12 @@ [fxShow]="!isEdit && !entity?.default"> {{'device-profile.set-default' | translate }} + + + + +
+
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+
+
+
+
+ + +
+
+ +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts new file mode 100644 index 0000000000..9d3affea42 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts @@ -0,0 +1,105 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; +import { ErrorStateMatcher } from '@angular/material/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@app/shared/components/dialog.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +export interface VcEntityExportDialogData { + entityId: EntityId; +} + +@Component({ + selector: 'tb-vc-entity-export-dialog', + templateUrl: './vc-entity-export-dialog.component.html', + providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], + styleUrls: [] +}) +export class VcEntityExportDialogComponent extends DialogComponent + implements OnInit, ErrorStateMatcher { + + exportFormGroup: FormGroup; + + submitted = false; + + createResult: VersionCreationResult; + + createResultMessage: SafeHtml; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, + @SkipSelf() private errorStateMatcher: ErrorStateMatcher, + public dialogRef: MatDialogRef, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private domSanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store, router, dialogRef); + + this.exportFormGroup = this.fb.group({ + branch: [null, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + ngOnInit(): void { + } + + isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { + const originalErrorState = this.errorStateMatcher.isErrorState(control, form); + const customErrorState = !!(control && control.invalid && this.submitted); + return originalErrorState || customErrorState; + } + + cancel(): void { + this.dialogRef.close(); + } + + export(): void { + this.submitted = true; + const request: SingleEntityVersionCreateRequest = { + entityId: this.data.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + this.createResult = result; + const message = this.translate.instant('version-control.export-entity-version-result-message', + {name: result.version.name, commitId: result.version.id}); + this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); + }); + } +} diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 6dc8746b2e..1730c69084 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,6 +22,11 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + VcEntityExportDialogComponent, + VcEntityExportDialogData +} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -41,6 +46,17 @@ export class HomeDialogsService { } } + public exportVcEntity(entityId: EntityId): Observable { + return this.dialog.open(VcEntityExportDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + entityId + } + }).afterClosed(); + } + private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { @@ -53,4 +69,5 @@ export class HomeDialogsService { } }).afterClosed(); } + } diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 00ed882cd9..75171b4532 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,6 +46,12 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} + + + + + + + + + + + + {{ 'version-control.branch-required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts new file mode 100644 index 0000000000..421710181a --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -0,0 +1,199 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable, of } from 'rxjs'; +import { + catchError, + debounceTime, + distinctUntilChanged, + map, + publishReplay, + refCount, + switchMap, + tap +} from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { BranchInfo } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; + +@Component({ + selector: 'tb-branch-autocomplete', + templateUrl: './branch-autocomplete.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => BranchAutocompleteComponent), + multi: true + }] +}) +export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { + + branchFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + @Input() + selectDefaultBranch = true; + + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; + + filteredBranches: Observable>; + + branches: Observable>; + + searchText = ''; + + private dirty = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + this.branchFormGroup = this.fb.group({ + branch: [null, []] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + + this.branches = null; + this.filteredBranches = this.branchFormGroup.get('branch').valueChanges + .pipe( + debounceTime(150), + distinctUntilChanged(), + tap(value => { + this.updateView(value); + }), + map(value => value ? value : ''), + switchMap(branch => this.fetchBranches(branch)) + ); + } + + ngAfterViewInit(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } + + selectDefaultBranchIfNeeded(): void { + if (this.selectDefaultBranch && !this.modelValue) { + this.getBranches().subscribe( + (data) => { + if (data && data.length) { + const defaultBranch = data.find(branch => branch.default); + if (defaultBranch) { + this.modelValue = defaultBranch.name; + this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); + this.propagateChange(this.modelValue); + } + } + } + ); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + if (value != null) { + this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + } else { + this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.selectDefaultBranchIfNeeded(); + } + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.branchFormGroup.get('branch').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + updateView(value: string | null) { + if (this.modelValue !== value) { + this.modelValue = value; + this.propagateChange(this.modelValue); + } + } + + displayBranchFn(branch?: string): string | undefined { + return branch ? branch : undefined; + } + + fetchBranches(searchText?: string): Observable> { + this.searchText = searchText; + return this.getBranches().pipe( + map(branches => branches.map(branch => branch.name).filter(branchName => { + return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; + })) + ); + } + + getBranches(): Observable> { + if (!this.branches) { + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + this.branches = branchesObservable.pipe( + catchError(() => of([] as Array)), + publishReplay(1), + refCount() + ); + } + return this.branches; + } + + clear() { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); + setTimeout(() => { + this.branchInput.nativeElement.blur(); + this.branchInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts new file mode 100644 index 0000000000..0c890ff9b4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -0,0 +1,55 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from '@shared/models/id/entity-id'; + +export interface VersionCreateConfig { + saveRelations: boolean; +} + +export enum VersionCreateRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + COMPLEX = 'COMPLEX' +} + +export interface VersionCreateRequest { + versionName: string; + branch: string; + type: VersionCreateRequestType; +} + +export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { + entityId: EntityId; + config: VersionCreateConfig; + type: VersionCreateRequestType.SINGLE_ENTITY; +} + +export interface BranchInfo { + name: string; + default: boolean; +} + +export interface EntityVersion { + id: string; + name: string; +} + +export interface VersionCreationResult { + version: EntityVersion; + added: number; + modified: number; + removed: number; +} diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 8d8de8e74a..7e73452b18 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -163,6 +163,7 @@ import { HtmlComponent } from '@shared/components/html.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { MultipleImageInputComponent } from '@shared/components/multiple-image-input.component'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -284,7 +285,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ], imports: [ CommonModule, @@ -484,7 +486,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5aaadb2bd6..00880644e4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3049,6 +3049,18 @@ "json-value-invalid": "JSON value has an invalid format", "json-value-required": "JSON value is required." }, + "version-control": { + "branch": "Branch", + "select-branch": "Select branch", + "branch-required": "Branch is required", + "export-entity-version": "Export entity version", + "entity-version-exported": "Entity version successfully exported", + "version-name": "Version name", + "version-name-required": "Version name is required", + "export-entity-relations": "Export entity relations", + "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", + "export-to-git": "Export to Git" + }, "widget": { "widget-library": "Widgets Library", "widget-bundle": "Widgets Bundle", From 421624f3d48f9a0793077582d68d3fdc98197657 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 12:41:14 +0300 Subject: [PATCH 073/178] Fixed sql test data --- dao/src/test/resources/sql/system-data.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 339d6e94ba..8e38b15010 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", From 4472bd7f7c3d70f7a75a3b26bd0a81769789ce6b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 23 May 2022 12:50:59 +0300 Subject: [PATCH 074/178] Initial implementation of the ClusterVersionControlService --- .../server/actors/ActorSystemContext.java | 2 +- .../DeviceProfileMsgConstructor.java | 2 +- .../queue/DefaultTbClusterService.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 13 +- .../DefaultTbRuleEngineConsumerService.java | 3 +- .../processing/AbstractConsumerService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 63 ++--- .../DefaultGitVersionControlQueueService.java | 79 ++++++- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 9 + .../service/sync/vc/PendingGitRequest.java | 1 - .../service/sync/vc/VoidGitRequest.java | 29 +++ .../transport/DefaultTransportApiService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 + common/cluster-api/src/main/proto/queue.proto | 24 +- .../vc/EntitiesVersionControlSettings.java | 7 +- common/queue/pom.xml | 4 + .../InMemoryMonolithQueueFactory.java | 13 +- .../util/DataDecodingEncodingService.java | 2 +- .../queue}/util/ProtoWithFSTService.java | 3 +- .../service/ProtoTransportEntityService.java | 5 +- common/transport/transport-api/pom.xml | 4 - .../DefaultTransportDeviceProfileCache.java | 2 +- .../DefaultTransportResourceCache.java | 2 +- .../service/DefaultTransportService.java | 2 +- .../DefaultTransportTenantProfileCache.java | 2 +- common/version-control/pom.xml | 12 +- .../sync/vc/ClusterVersionControlService.java | 22 ++ .../DefaultClusterVersionControlService.java | 219 ++++++++++++++++++ .../sync/vc/GitVersionControlService.java | 52 ----- .../sync/vc/VersionControlRequestCtx.java | 49 ++++ 31 files changed, 505 insertions(+), 134 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/DataDecodingEncodingService.java (93%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/ProtoWithFSTService.java (93%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java delete mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 500ad60524..733cf75e64 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -48,7 +48,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java index 77dc121e38..7943ec0595 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java @@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 4a52ea2287..6b475f5a5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -140,6 +140,7 @@ public class DefaultTbClusterService implements TbClusterService { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + //TODO: ashvayka toCoreMsgs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index bec0dceed3..4cf7f86097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -74,6 +74,8 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -116,6 +118,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; @@ -137,7 +140,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { - if (!request.getFuture().isDone()) { - pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); - } else { - throw new RuntimeException("Future is already done!"); - } - } - @Override public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); @@ -218,6 +214,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu // } } + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + @Override public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -233,6 +238,56 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return request.getFuture(); } + @Override + public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture clearRepository(TenantId tenantId) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public void processResponse(VersionControlResponseMsg vcResponseMsg) { + UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); + PendingGitRequest request = pendingRequestMap.get(requestId); + if (request == null) { + log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); + return; + } else { + log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + } + var future = request.getFuture(); + if (!StringUtils.isEmpty(vcResponseMsg.getError())) { + future.setException(new RuntimeException(vcResponseMsg.getError())); + } else { + if (vcResponseMsg.hasGenericResponse()) { + future.set(null); + } + } + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 02afafedad..bb42e44f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; +import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -57,8 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); - void deleteVersionControlSettings(TenantId tenantId); + void deleteVersionControlSettings(TenantId tenantId) throws Exception; - void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index e7b7171ea0..f09b383d87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -21,10 +21,12 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import java.util.List; @@ -54,4 +56,11 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture clearRepository(TenantId tenantId); + + void processResponse(VersionControlResponseMsg vcResponseMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 3650f9e113..09bc272828 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; -import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java new file mode 100644 index 0000000000..d829960ab4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; + +import java.util.List; + +public class VoidGitRequest extends PendingGitRequest { + + public VoidGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6c4adfc5ae..f8b42ecc1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1032c79054..3fb48d2789 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1024,6 +1024,8 @@ queue: vc: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index cb1a1ab4db..7a5030277e 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -745,24 +745,34 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message GenericRepositoryRequestMsg {} + +message GenericRepositoryResponseMsg {} + message ToVersionControlServiceMsg { string nodeId = 1; int64 tenantIdMSB = 2; int64 tenantIdLSB = 3; int64 requestIdMSB = 4; int64 requestIdLSB = 5; - CommitRequestMsg commitRequest = 6; - ListVersionsRequestMsg listVersionRequest = 7; - ListEntitiesRequestMsg listEntitiesRequest = 8; - ListBranchesRequestMsg listBranchesRequest = 9; - EntityContentRequestMsg entityContentRequest = 10; - EntitiesContentRequestMsg entitiesContentRequest = 11; + bytes vcSettings = 6; + GenericRepositoryRequestMsg initRepositoryRequest = 7; + GenericRepositoryRequestMsg testRepositoryRequest = 8; + GenericRepositoryRequestMsg clearRepositoryRequest = 9; + CommitRequestMsg commitRequest = 10; + ListVersionsRequestMsg listVersionRequest = 11; + ListEntitiesRequestMsg listEntitiesRequest = 12; + ListBranchesRequestMsg listBranchesRequest = 13; + EntityContentRequestMsg entityContentRequest = 14; + EntitiesContentRequestMsg entitiesContentRequest = 15; } message VersionControlResponseMsg { int64 requestIdMSB = 1; int64 requestIdLSB = 2; - CommitResponseMsg commitResponse = 3; + string error = 3; + GenericRepositoryResponseMsg genericResponse = 4; + CommitResponseMsg commitResponse = 5; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..bc098fbe1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,8 +16,13 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + +import java.io.Serializable; + @Data -public class EntitiesVersionControlSettings { +public class EntitiesVersionControlSettings implements Serializable { + private static final long serialVersionUID = -3211552851889198721L; + private String repositoryUri; private VersionControlAuthMethod authMethod; private String username; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e2197d566f..a625ec6e93 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,6 +120,10 @@ com.google.protobuf protobuf-java-util + + de.ruedigermoeller + fst + org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 98c1a8029e..b6b89fcc2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -37,28 +37,32 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, InMemoryStorage storage) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; + this.vcSettings = vcSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -91,6 +95,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, vcSettings.getTopic()); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new InMemoryTbQueueConsumer<>(storage, configuration.getTopic()); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java index ed0572e8e5..8ebcc39cab 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import java.util.Optional; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index 1c5eec383c..f1c55973db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import java.util.Optional; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index c75de25267..a10c3e4969 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Service diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8b3f9feb22..51373e2a4a 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -64,10 +64,6 @@ com.google.code.gson gson - - de.ruedigermoeller - fst - org.slf4j slf4j-api diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index b79306199f..2b0f73343d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java index 666941b143..b4187432e8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.TransportResourceCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 5b29d3c3ff..3256039e9a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -69,7 +69,7 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4cfefb7a6f..bb7f8f3f4d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 7b46f7ba72..294141db8f 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -36,6 +36,14 @@ + + org.thingsboard.common + data + + + org.thingsboard.common + queue + org.springframework spring-core @@ -105,10 +113,6 @@ awaitility test - - org.thingsboard.common - data - diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java new file mode 100644 index 0000000000..fc134e1850 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.springframework.context.ApplicationListener; + +public interface ClusterVersionControlService extends ApplicationListener { +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java new file mode 100644 index 0000000000..73089acdf5 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -0,0 +1,219 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbVersionControlComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +@TbVersionControlComponent +@Service +@RequiredArgsConstructor +public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final TbVersionControlQueueFactory queueFactory; + private final DataDecodingEncodingService encodingService; + private final GitRepositoryService vcService; + private final NotificationsTopicService notificationsTopicService; + + private volatile ExecutorService consumerExecutor; + private volatile TbQueueConsumer> consumer; + private volatile TbQueueProducer> producer; + private volatile boolean stopped = false; + + @Value("${queue.vc.poll-interval:25}") + private long pollDuration; + @Value("${queue.vc.pack-processing-timeout:60000}") + private long packProcessingTimeout; + + @PostConstruct + public void init() { + consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); + producer = queueFactory.createTbCoreNotificationsMsgProducer(); + consumer = queueFactory.createToVersionControlMsgConsumer(); + } + + @PreDestroy + public void stop() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + //TODO: cleanup repositories that we no longer manage in this node. + consumer.subscribe(event.getPartitions()); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + consumerExecutor.execute(() -> consumerLoop(consumer)); + } + + void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msgWrapper : msgs) { + ToVersionControlServiceMsg msg = msgWrapper.getValue(); + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(new VersionControlRequestCtx(msg, null)); + } else { + VersionControlRequestCtx ctx = new VersionControlRequestCtx(msg, getEntitiesVersionControlSettings(msg)); + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } + } + } +// ConcurrentMap> pendingMap = msgs.stream().collect( +// Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); +// CountDownLatch processingTimeoutLatch = new CountDownLatch(1); +// TbPackProcessingContext> ctx = new TbPackProcessingContext<>( +// processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); +// pendingMap.forEach((id, msg) -> { +// log.trace("[{}] Creating downlink callback for message: {}", id, msg.getValue()); +// TbCallback callback = new TbPackCallback<>(id, ctx); +// try { +// handleDownlink(id, msg, callback); +// } catch (Throwable e) { +// log.warn("[{}] Failed to process notification: {}", id, msg, e); +// callback.onFailure(e); +// } +// }); +// if (!processingTimeoutLatch.await(processingTimeout, TimeUnit.MILLISECONDS)) { +// ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process downlink: {}", id, msg.getValue())); +// ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process downlink: {}", id, msg.getValue())); +// } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain version control requests from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new version control messages", e2); + } + } + } + } + log.info("TB Version Control request consumer stopped."); + } + + private void handleClearRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.clearRepository(ctx.getTenantId()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void handleInitRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + + private void handleTestRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.testRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void reply(VersionControlRequestCtx ctx, Optional e) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); + TransportProtos.VersionControlResponseMsg.Builder builder = TransportProtos.VersionControlResponseMsg.newBuilder() + .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) + .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); + if (e.isPresent()) { + builder.setError(e.get().getMessage()); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); + log.trace("PUSHING msg: {} to: {}", msg, tpi); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + if (settingsOpt.isPresent()) { + return settingsOpt.get(); + } else { + log.warn("Failed to parse VC settings: {}", msg.getVcSettings()); + throw new RuntimeException("Failed to parse vc settings!"); + } + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java deleted file mode 100644 index 56349cf8a5..0000000000 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; - -import java.util.List; - -public interface GitVersionControlService { - - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void clearRepository(TenantId tenantId); - - List listVersions(TenantId tenantId, String branch); - - List listVersions(TenantId tenantId, String branch, EntityType entityType); - - List listVersions(TenantId tenantId, String branch, EntityId entityId); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); - - List listBranches(TenantId tenantId); - - EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); - - List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); - -} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java new file mode 100644 index 0000000000..6541a2978a --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; + +import java.util.UUID; + +@RequiredArgsConstructor +@Data +public class VersionControlRequestCtx { + private final String nodeId; + private final UUID requestId; + private final TenantId tenantId; + private final EntitiesVersionControlSettings settings; + + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + this.nodeId = msg.getNodeId(); + this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); + this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + this.settings = settings; + } + + @Override + public String toString() { + return "VersionControlRequestCtx{" + + "nodeId='" + nodeId + '\'' + + ", requestId=" + requestId + + ", tenantId=" + tenantId + + '}'; + } +} From 2a89fc67aea28844c65444e3da579963c4a56254 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 13:20:00 +0300 Subject: [PATCH 075/178] Git api usage improvements --- .../DefaultEntitiesVersionControlService.java | 29 ++++++----- .../vc/LocalGitVersionControlService.java | 20 +++++++- .../src/main/resources/thingsboard.yml | 1 + .../vc/EntitiesVersionControlSettings.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 51 ++++++++++--------- .../server/service/sync/vc/GitRepository.java | 45 ++++++++-------- .../service/sync/vc/GitRepositoryService.java | 2 + .../server/service/sync/vc/PendingCommit.java | 2 + 8 files changed, 93 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d46884ca1e..4aedfb9699 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.*; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; - -import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; -import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import java.util.ArrayList; import java.util.HashMap; @@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..fa767bd1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService { if (old != null) { gitRepositoryService.abort(old); } - gitRepositoryService.prepareCommit(pendingCommit); + try { + gitRepositoryService.prepareCommit(pendingCommit); + } catch (Exception e) { + pendingCommitMap.remove(tenantId); + gitRepositoryService.cleanUp(pendingCommit); + throw e; + } return pendingCommit; } finally { lock.unlock(); @@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { @Override public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); + VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push); + pendingCommitMap.remove(commit.getTenantId()); + return result; } @Override @@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); r.accept(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } @@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); return c.apply(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 654929f500..115b4d2ebd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1119,6 +1119,7 @@ vc: git: service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: endpoints: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..308f412228 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} \ No newline at end of file +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index aac40e0dcb..e85dee8e2b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; @Service public class DefaultGitRepositoryService implements GitRepositoryService { - @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + @Value("${vc.git.repositories-folder}") private String repositoriesFolder; @Value("${vc.git.repos-poll-interval:60}") @@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService { String branch = commit.getRequest().getBranch(); try { repository.fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout("origin/" + branch, false); - try { - repository.checkout(branch, true); - } catch (RefAlreadyExistsException e) { - repository.checkout(branch, false); - } + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { repository.merge(branch); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } } } catch (IOException | GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; @@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setRemoved(status.getRemoved().size()); GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); - repository.push(); + repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch()); result.setVersion(toVersion(gitCommit)); return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); + } + } + + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); } @Override public void abort(PendingCommit commit) { - //TODO: implement; + cleanUp(commit); } @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { - GitRepository repository = checkRepository(tenantId); - return repository.getFileContentAtCommit(relativePath, versionId); + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); } @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { - return repository.listBranches(); + return repository.listRemoteBranches(); } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index fa4891d35a..5f1a13ec53 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.sshd.JGitKeyCache; @@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -116,15 +124,18 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch, boolean createBranch) throws GitAPIException { - execute(git.checkout() - .setCreateBranch(createBranch) - .setName(branch)); + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); } - public void reset() throws GitAPIException { + public void resetAndClean() throws GitAPIException { execute(git.reset() .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); } public void merge(String branch) throws IOException, GitAPIException { @@ -137,9 +148,9 @@ public class GitRepository { } - public List listBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(name -> StringUtils.removeStart(name, "origin/")) @@ -209,16 +220,9 @@ public class GitRepository { .setOrphan(true) .setForced(true) .setName(name)); -// Set uncommittedChanges = git.status().call().getUncommittedChanges(); -// if (!uncommittedChanges.isEmpty()) { -// RmCommand rm = git.rm(); -// uncommittedChanges.forEach(rm::addFilepattern); -// execute(rm); -// } -// execute(git.clean()); } - public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + public void add(String filesPattern) throws GitAPIException { execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); } @@ -230,13 +234,14 @@ public class GitRepository { public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() - .setMessage(message)); // TODO [viacheslav]: set configurable author for commit + .setMessage(message)); return toCommit(revCommit); } - public void push() throws GitAPIException { - execute(git.push()); + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 19754750c2..bbdf976804 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -44,6 +44,8 @@ public interface GitRepositoryService { VersionCreationResult push(PendingCommit commit); + void cleanUp(PendingCommit commit); + void abort(PendingCommit commit); List listBranches(TenantId tenantId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index a30f7514a2..16f3ce5e40 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -27,10 +27,12 @@ public class PendingCommit { private final UUID txId; private final TenantId tenantId; private final VersionCreateRequest request; + private final String workingBranch; public PendingCommit(TenantId tenantId, VersionCreateRequest request) { this.txId = UUID.randomUUID(); this.tenantId = tenantId; this.request = request; + this.workingBranch = txId.toString(); } } From 55966e033e75e7e7d5d13f4d55452f4698f76189 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 14:04:12 +0300 Subject: [PATCH 076/178] Fixes for vc api --- .../EntitiesVersionControlController.java | 2 +- .../sync/vc/LocalGitVersionControlService.java | 14 ++++++-------- .../common/data/sync/ie/EntityExportData.java | 3 ++- .../common/data/sync/ie/EntityImportResult.java | 6 ------ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..39a8dc464f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -223,7 +223,7 @@ public class EntitiesVersionControlController extends BaseController { if (versionId == null) { List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); if (versions.size() > 0) { - versionId = versions.get(0).getId(); + request.setVersionId(versions.get(0).getId()); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..8aa4b793df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -16,21 +16,17 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +34,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -62,13 +57,16 @@ import java.util.stream.Collectors; @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) public class LocalGitVersionControlService implements GitVersionControlService { - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); + private final ObjectMapper jsonMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { @@ -165,7 +163,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { doInsideLock(commit, c -> { String entityDataJson; try { - entityDataJson = jsonWriter.writeValueAsString(entityData); + entityDataJson = jsonMapper.writeValueAsString(entityData); gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), entityData.getEntity().getId().toString()), entityDataJson); } catch (IOException e) { @@ -230,7 +228,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + return jsonMapper.readValue(entityDataJson, EntityExportData.class); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index b361a302fc..35fceafe43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index ea69425a2c..8d6d9ff7b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -15,26 +15,20 @@ */ package org.thingsboard.server.common.data.sync.ie; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.JsonTbEntity; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { - @JsonTbEntity private E savedEntity; - @JsonTbEntity private E oldEntity; private EntityType entityType; - @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @JsonIgnore private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { From 2dcdb4af8e561fedd600b8af5cd86544251e91c1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 15:36:57 +0300 Subject: [PATCH 077/178] Fix Entities Version Control controller - add TbCoreComponent annotation --- .../server/controller/EntitiesVersionControlController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..a2b0bf3859 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -41,6 +42,7 @@ import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @RestController +@TbCoreComponent @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor From 0d57f2f6c2baf8200066a6c34b67d2bdb87a97ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 18:19:07 +0300 Subject: [PATCH 078/178] Use page link to get versions list --- .../server/controller/AdminController.java | 14 ++++++ .../EntitiesVersionControlController.java | 50 +++++++++++++------ .../DefaultEntitiesVersionControlService.java | 26 +++++----- .../vc/EntitiesVersionControlService.java | 8 +-- .../vc/LocalGitVersionControlService.java | 18 ++++--- common/version-control/pom.xml | 1 - .../sync/vc/DefaultGitRepositoryService.java | 10 ++-- .../server/service/sync/vc/GitRepository.java | 44 ++++++++++------ .../service/sync/vc/GitRepositoryService.java | 4 +- .../sync/vc/GitVersionControlService.java | 8 +-- 10 files changed, 118 insertions(+), 65 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 45b8aa5f95..ae9bf18bcf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -203,6 +203,20 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings/exists") + @ResponseBody + public Boolean versionControlSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index a2b0bf3859..4086788fea 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -25,6 +26,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @@ -39,7 +42,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -109,13 +112,18 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public PageData listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink); } catch (Exception e) { throw handleException(e); } @@ -128,11 +136,16 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public PageData listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink); } catch (Exception e) { throw handleException(e); } @@ -153,10 +166,15 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public PageData listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listVersions(getTenantId(), branch, pageLink); } catch (Exception e) { throw handleException(e); } @@ -223,9 +241,9 @@ public class EntitiesVersionControlController extends BaseController { try { String versionId = request.getVersionId(); if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); + PageData versions = versionControlService.listVersions(user.getTenantId(), request.getBranch(), new PageLink(1)); + if (versions.getData().size() > 0) { + versionId = versions.getData().get(0).getId(); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 4aedfb9699..8f0d11068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -134,18 +136,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, externalId, pageLink); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, entityType, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, pageLink); } @Override @@ -307,6 +309,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont adminSettings.setKey(SETTINGS_KEY); adminSettings.setTenantId(tenantId); } + try { + gitService.clearRepository(tenantId); + gitService.initRepository(tenantId, versionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -315,12 +323,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } return savedVersionControlSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..a70b78aaf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -37,11 +39,11 @@ public interface EntitiesVersionControlService { VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..42a1308901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -31,6 +31,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -183,18 +185,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) { + return listVersions(tenantId, branch, (String) null, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + public PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityType, null), pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + public PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), pageLink); } @Override @@ -249,9 +251,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { return null; } - private List listVersions(TenantId tenantId, String branch, String path) { + private PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) { try { - return gitRepositoryService.listVersions(tenantId, branch, path); + return gitRepositoryService.listVersions(tenantId, branch, path, pageLink); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index b01b466cb5..89c36b1c96 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -60,7 +60,6 @@ com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index e85dee8e2b..b822ff3b5b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -178,7 +180,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() + return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() .filter(version -> version.getId().equals(versionId)) .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); } @@ -189,11 +191,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); } @Override diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5f1a13ec53..22be22d991 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.LsRemoteCommand; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -45,6 +39,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -59,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class GitRepository { @@ -157,27 +154,25 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { - return listCommits(branch, null, limit); + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); } - public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } LogCommand command = git.log() - .add(branchId).setMaxCount(limit) + .add(branchId) .setRevFilter(RevFilter.NO_MERGES); if (StringUtils.isNotEmpty(path)) { command.addPath(path); } - return Streams.stream(execute(command)) - .map(this::toCommit) - .collect(Collectors.toList()); + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink); } - public List listFilesAtCommit(String commitId) throws IOException { return listFilesAtCommit(commitId, null); } @@ -295,6 +290,23 @@ public class GitRepository { return command.call(); } + private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { if (credentialsProvider != null) { transportCommand.setCredentialsProvider(credentialsProvider); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index bbdf976804..8fb0e0f84d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -28,7 +30,7 @@ public interface GitRepositoryService { void prepareCommit(PendingCommit pendingCommit); - List listVersions(TenantId tenantId, String branch, String path) throws Exception; + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..c21efbb774 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -44,11 +46,11 @@ public interface GitVersionControlService { VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityType entityType); + PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityId entityId); + PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); From 1065fd9fbebec782ed7fddc268f26d712cd10917 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 11:38:03 +0300 Subject: [PATCH 079/178] Implementation of the API --- .../DefaultGitVersionControlQueueService.java | 49 ++- .../vc/LocalGitVersionControlService.java | 290 ------------------ .../service/sync/vc/PendingGitRequest.java | 4 +- common/cluster-api/src/main/proto/queue.proto | 37 +++ .../data/sync/vc/VersionedEntityInfo.java | 6 + .../DefaultClusterVersionControlService.java | 109 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../service/sync/vc/GitRepositoryService.java | 4 +- 8 files changed, 188 insertions(+), 319 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7ffcbb9c14..0d3f2a4431 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -21,17 +21,16 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -61,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.function.Function; +import java.util.stream.Collectors; @TbCoreComponent @Service @@ -209,6 +209,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override + @SuppressWarnings("rawtypes") public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() @@ -218,14 +219,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() , wrap(request.getFuture())); return request.getFuture(); -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } } private void registerAndSend(PendingGitRequest request, @@ -237,13 +230,16 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request, settings)), callback); + var requestBody = enrichFunction.apply(newRequestProto(request, settings)); + log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); + clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); } else { throw new RuntimeException("Future is already done!"); } } @Override + @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -313,10 +309,41 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); ((CommitGitRequest) request).getFuture().set(commitResult); + } else if (vcResponseMsg.hasListBranchesResponse()) { + var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + } else if (vcResponseMsg.hasListEntitiesResponse()) { + var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); + ((ListEntitiesGitRequest) request).getFuture().set( + listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListVersionsResponse()) { + var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); + ((ListVersionsGitRequest) request).getFuture().set( + listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList())); + } else if (vcResponseMsg.hasEntityContentResponse()) { + var data = vcResponseMsg.getEntityContentResponse().getData(); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else if (vcResponseMsg.hasEntitiesContentResponse()) { + var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); + ((EntitiesContentGitRequest) request).getFuture() + .set(dataList.stream().map(this::toData).collect(Collectors.toList())); } } } + private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { + return new EntityVersion(proto.getId(), proto.getName()); + } + + private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { + return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); + } + + @SuppressWarnings("rawtypes") + private EntityExportData toData(String data) { + return JacksonUtil.fromString(data, EntityExportData.class); + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java deleted file mode 100644 index d995472531..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; - -import java.io.IOException; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -@RequiredArgsConstructor -@Service -@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService { - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private final GitRepositoryService gitRepositoryService; - private final TenantDao tenantDao; - private final AdminSettingsService adminSettingsService; - private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); -// -// @AfterStartUp -// public void init() { -// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { -// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); -// if (settings != null) { -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// log.warn("Failed to init repository for tenant {}", tenantId, e); -// } -// } -// }); -// } -// -// @Override -// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.testRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void clearRepository(TenantId tenantId) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.clearRepository(tenantId); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// var pendingCommit = new PendingCommit(tenantId, request); -// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); -// if (old != null) { -// gitRepositoryService.abort(old); -// } -// gitRepositoryService.prepareCommit(pendingCommit); -// return pendingCommit; -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void deleteAll(PendingCommit commit, EntityType entityType) { -// doInsideLock(commit, c -> { -// try { -// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { -// doInsideLock(commit, c -> { -// String entityDataJson; -// try { -// entityDataJson = jsonWriter.writeValueAsString(entityData); -// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), -// entityData.getEntity().getId().toString()), entityDataJson); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public VersionCreationResult push(PendingCommit commit) { -// return executeInsideLock(commit, gitRepositoryService::push); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch) { -// return listVersions(tenantId, branch, (String) null); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { -// return listVersions(tenantId, branch, getRelativePath(entityType, null)); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { -// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { -// try { -// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { -// return listEntitiesAtVersion(tenantId, branch, versionId, null); -// } -// -// @Override -// public List listBranches(TenantId tenantId) { -// return gitRepositoryService.listBranches(tenantId); -// } -// -// @Override -// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { -// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() -// .skip(offset).limit(limit) -// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) -// .collect(Collectors.toList()); -// } -// -// @Override -// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { -// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); -// if (adminSettings != null) { -// try { -// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); -// } catch (Exception e) { -// throw new RuntimeException("Failed to load version control settings!", e); -// } -// } -// return null; -// } -// -// private List listVersions(TenantId tenantId, String branch, String path) { -// try { -// return gitRepositoryService.listVersions(tenantId, branch, path); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private void doInsideLock(PendingCommit commit, Consumer r) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// r.accept(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private T executeInsideLock(PendingCommit commit, Function c) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// return c.apply(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private void checkCommit(PendingCommit commit) { -// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); -// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { -// throw new ConcurrentModificationException(); -// } -// } -// -// private String getRelativePath(EntityType entityType, String entityId) { -// String path = entityType.name().toLowerCase(); -// if (entityId != null) { -// path += "/" + entityId + ".json"; -// } -// return path; -// } -// -// private Lock getRepoLock(TenantId tenantId) { -// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); -// } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 9b612cee5b..e63ba04f45 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -24,17 +24,19 @@ import java.util.UUID; @Getter public class PendingGitRequest { + private final long createdTime; private final UUID requestId; private final TenantId tenantId; private final SettableFuture future; public PendingGitRequest(TenantId tenantId) { + this.createdTime = System.currentTimeMillis(); this.requestId = UUID.randomUUID(); this.tenantId = tenantId; this.future = SettableFuture.create(); } - public boolean requiresSettings(){ + public boolean requiresSettings() { return true; } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 6d45698451..54e46eebcc 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -723,15 +723,39 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; } +message EntityVersionProto { + int64 ts = 1; + string id = 2; + string name = 3; +} + +message ListVersionsResponseMsg { + repeated EntityVersionProto versions = 1; +} + message ListEntitiesRequestMsg { string branchName = 1; string versionId = 2; string entityType = 3; } +message VersionedEntityInfoProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; +} + +message ListEntitiesResponseMsg { + repeated VersionedEntityInfoProto entities = 1; +} + message ListBranchesRequestMsg { } +message ListBranchesResponseMsg { + repeated string branches = 1; +} + message EntityContentRequestMsg { string versionId = 1; string entityType = 2; @@ -739,6 +763,10 @@ message EntityContentRequestMsg { int64 entityIdLSB = 4; } +message EntityContentResponseMsg { + string data = 1; +} + message EntitiesContentRequestMsg { string versionId = 1; string entityType = 2; @@ -746,6 +774,10 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message EntitiesContentResponseMsg { + repeated string data = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -774,6 +806,11 @@ message VersionControlResponseMsg { string error = 3; GenericRepositoryResponseMsg genericResponse = 4; CommitResponseMsg commitResponse = 5; + ListBranchesResponseMsg listBranchesResponse = 6; + ListEntitiesResponseMsg listEntitiesResponse = 7; + ListVersionsResponseMsg listVersionsResponse = 8; + EntityContentResponseMsg entityContentResponse = 9; + EntitiesContentResponseMsg entitiesContentResponse = 10; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 163fe4c6d2..fd278cde1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -16,10 +16,16 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; @Data +@NoArgsConstructor public class VersionedEntityInfo { private EntityId externalId; // etc.. + + public VersionedEntityInfo(EntityId externalId) { + this.externalId = externalId; + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b9c595beac..b282799e24 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -23,9 +23,12 @@ import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -58,6 +61,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @TbVersionControlComponent @@ -144,9 +148,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var newSettings = ctx.getSettings(); if (!newSettings.equals(currentSettings)) { vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } else { + vcService.fetch(ctx.getTenantId()); } if (msg.hasCommitRequest()) { handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); } } } @@ -172,12 +188,71 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } - private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg commitRequest) throws Exception { + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { + var entityType = EntityType.valueOf(request.getEntityType()); + String path = getRelativePath(entityType, null); + var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) + .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); + var response = EntitiesContentResponseMsg.newBuilder(); + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + response.addData(data); + } + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + } + + private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { + String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + } + + private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { + String path; + if (StringUtils.isNotEmpty(request.getEntityType())) { + var entityType = EntityType.valueOf(request.getEntityType()); + if (request.getEntityIdLSB() != 0 || request.getEntityIdMSB() != 0) { + path = getRelativePath(entityType, new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + } else { + path = getRelativePath(entityType, null); + } + } else { + path = null; + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() + .addAllVersions(data.stream().map( + v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + ).collect(Collectors.toList())))); + } + + private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { + EntityType entityType = StringUtils.isNotEmpty(request.getEntityType()) ? EntityType.valueOf(request.getEntityType()) : null; + var path = entityType != null ? getRelativePath(entityType, null) : null; + var data = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListEntitiesResponse(ListEntitiesResponseMsg.newBuilder() + .addAllEntities(data.stream().map(VersionedEntityInfo::getExternalId).map( + id -> VersionedEntityInfoProto.newBuilder() + .setEntityType(id.getEntityType().name()) + .setEntityIdMSB(id.getId().getMostSignificantBits()) + .setEntityIdLSB(id.getId().getLeastSignificantBits()).build() + ).collect(Collectors.toList())))); + } + + private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { + var branches = vcService.listBranches(ctx.getTenantId()); + reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); + } + + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); - UUID txId = UUID.fromString(commitRequest.getTxId()); - if (commitRequest.hasPrepareMsg()) { - prepareCommit(ctx, txId, commitRequest.getPrepareMsg()); - } else if (commitRequest.hasAbortMsg()) { + UUID txId = UUID.fromString(request.getTxId()); + if (request.hasPrepareMsg()) { + prepareCommit(ctx, txId, request.getPrepareMsg()); + } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { doAbortCurrentCommit(tenantId, current); @@ -186,11 +261,11 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { try { - if (commitRequest.hasAddMsg()) { - addToCommit(ctx, current, commitRequest.getAddMsg()); - } else if (commitRequest.hasDeleteMsg()) { - deleteFromCommit(ctx, current, commitRequest.getDeleteMsg()); - } else if (commitRequest.hasPushMsg()) { + if (request.hasAddMsg()) { + addToCommit(ctx, current, request.getAddMsg()); + } else if (request.hasDeleteMsg()) { + deleteFromCommit(ctx, current, request.getDeleteMsg()); + } else if (request.hasPushMsg()) { reply(ctx, vcService.push(current)); } } catch (Exception e) { @@ -198,7 +273,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe throw e; } } else { - log.debug("[{}] Ignore request due to stale commit: {}", txId, commitRequest); + log.debug("[{}] Ignore request due to stale commit: {}", txId, request); } } } @@ -290,7 +365,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); } ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); - log.trace("PUSHING msg: {} to: {}", msg, tpi); + log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @@ -304,9 +379,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } - - } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 0d55783a31..758641f6eb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -160,6 +160,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { //TODO: implement; } + @Override + public void fetch(TenantId tenantId) { + //Fetch latest changes on demand. + } + @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { GitRepository repository = checkRepository(tenantId); @@ -197,9 +202,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + public List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index c748147856..54ab91582e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -30,7 +30,7 @@ public interface GitRepositoryService { List listVersions(TenantId tenantId, String branch, String path) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; @@ -51,4 +51,6 @@ public interface GitRepositoryService { List listBranches(TenantId tenantId); String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + + void fetch(TenantId tenantId); } From a5ff23a0a479275f35f4e443497f78daf0eec4d0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 24 May 2022 14:03:53 +0300 Subject: [PATCH 080/178] Add commit timestamp and timestamp sorting --- .../EntitiesVersionControlController.java | 29 +++++++++++----- .../DefaultGitVersionControlQueueService.java | 4 +-- common/cluster-api/src/main/proto/queue.proto | 11 ++++--- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 33 +++++++++++++++---- 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index ef1d402c42..0efa52dee2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -53,9 +53,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; @RestController @TbCoreComponent @@ -131,10 +130,14 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); } catch (Exception e) { throw handleException(e); @@ -154,9 +157,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); } catch (Exception e) { throw handleException(e); @@ -183,9 +190,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink)); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fb6ab3e0d6..fbee0502e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -312,7 +312,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -344,7 +344,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index bd8e220a18..a1cbddf6e8 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -689,11 +689,12 @@ message CommitRequestMsg { } message CommitResponseMsg { - string commitId = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; + int64 ts = 1; + string commitId = 2; + string name = 3; + int32 added = 4; + int32 modified = 5; + int32 removed = 6; } message PrepareMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 8779d64ed3..50d3fef8a8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -23,6 +23,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor public class EntityVersion { + private long timestamp; private String id; private String name; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..47d0aca292 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -245,7 +245,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() ).collect(Collectors.toList()))) ); } @@ -364,6 +364,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() + .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) .setAdded(result.getAdded()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..8d711c3e69 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -259,7 +259,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } private EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dcefe7bed7..6208b913f4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; @@ -48,6 +49,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -58,10 +60,7 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PublicKey; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -180,7 +179,7 @@ public class GitRepository { command.addPath(path); } Iterable commits = execute(command); - return iterableToPageData(commits, this::toCommit, pageLink); + return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); } public List listFilesAtCommit(String commitId) throws IOException { @@ -282,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -300,11 +299,30 @@ public class GitRepository { return command.call(); } - private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + private static Function> revCommitComparatorFunction = pageLink -> { + SortOrder sortOrder = pageLink.getSortOrder(); + if (sortOrder != null + && sortOrder.getProperty().equals("timestamp") + && SortOrder.Direction.ASC.equals(sortOrder.getDirection())) { + return Comparator.comparingInt(RevCommit::getCommitTime); + } + return null; + }; + + private static PageData iterableToPageData (Iterable iterable, + Function mapper, + PageLink pageLink, + Function> comparatorFunction) { int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); int limit = startIndex + pageLink.getPageSize(); + if (comparatorFunction != null) { + Comparator comparator = comparatorFunction.apply(pageLink); + if (comparator != null) { + iterable = Ordering.from(comparator).immutableSortedCopy(iterable); + } + } iterable = Iterables.limit(iterable, limit); if (startIndex < totalElements) { iterable = Iterables.skip(iterable, startIndex); @@ -374,6 +392,7 @@ public class GitRepository { @Data public static class Commit { + private final long timestamp; private final String id; private final String message; private final String authorName; From 9deb67afe7ecddfc7ad72c89430c1a6cb959cde1 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 17:32:29 +0300 Subject: [PATCH 081/178] Version Control microservice --- .../src/main/resources/thingsboard.yml | 8 +- .../queue/discovery/ZkDiscoveryService.java | 2 - .../queue/settings/TbQueueCoreSettings.java | 2 + .../TbQueueRemoteJsInvokeSettings.java | 2 + .../settings/TbQueueRuleEngineSettings.java | 2 + .../settings/TbQueueTransportApiSettings.java | 2 + .../TbQueueTransportNotificationSettings.java | 2 + .../TbQueueVersionControlSettings.java | 2 + .../DefaultClusterVersionControlService.java | 156 ++++++++++---- .../sync/vc/DefaultGitRepositoryService.java | 37 ++-- .../service/sync/vc/GitRepositoryService.java | 3 + msa/pom.xml | 1 + msa/vc-executor-docker/docker/Dockerfile | 33 +++ .../docker/start-tb-vc-executor.sh | 33 +++ msa/vc-executor-docker/pom.xml | 190 ++++++++++++++++++ msa/vc-executor/pom.xml | 6 +- ...oardVersionControlExecutorApplication.java | 6 +- ...VersionControlQueueRoutingInfoService.java | 31 +++ ...ersionControlTenantRoutingInfoService.java | 30 +++ .../src/main/resources/logback.xml | 1 + .../src/main/resources/tb-vc-executor.yml | 160 ++++++++++++++- 21 files changed, 632 insertions(+), 77 deletions(-) create mode 100644 msa/vc-executor-docker/docker/Dockerfile create mode 100755 msa/vc-executor-docker/docker/start-tb-vc-executor.sh create mode 100644 msa/vc-executor-docker/pom.xml create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 76cc62d09a..aaec0852ca 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1025,7 +1025,7 @@ queue: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1120,9 +1120,11 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: - thread_pool_size: "${TB_VC_POOL_SIZE:4}" + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" git: - repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 8afd4dc768..16af4befd4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,14 +33,12 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 33e6552eb9..f64143b14c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueCoreSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index b930d04312..f4d62bad1d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRemoteJsInvokeSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eee70e9e22..57131f6b62 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 0ad9414bd8..a4dfe86bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportApiSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index d4c0064692..e2a10b31b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportNotificationSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java index e7f0ce2259..3a7a5602d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueVersionControlSettings { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..c4bb637eb0 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -15,8 +15,14 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; @@ -57,9 +63,10 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbVersionControlComponent; @@ -67,6 +74,7 @@ import org.thingsboard.server.queue.util.TbVersionControlComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +84,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -87,7 +97,8 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { - private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; private final TbVersionControlQueueFactory queueFactory; private final DataDecodingEncodingService encodingService; private final GitRepositoryService vcService; @@ -105,11 +116,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private long pollDuration; @Value("${queue.vc.pack-processing-timeout:60000}") private long packProcessingTimeout; + @Value("${vc.git.io_pool_size:3}") + private int ioPoolSize; + + //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. + private final List ioThreads = new ArrayList<>(); + @PostConstruct public void init() { consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); - producer = queueFactory.createTbCoreNotificationsMsgProducer(); + var threadFactory = ThingsBoardThreadFactory.forName("vc-io-thread"); + for (int i = 0; i < ioPoolSize; i++) { + ioThreads.add(MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(threadFactory))); + } + producer = producerProvider.getTbCoreNotificationsMsgProducer(); consumer = queueFactory.createToVersionControlMsgConsumer(); } @@ -122,11 +143,25 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe if (consumerExecutor != null) { consumerExecutor.shutdownNow(); } + ioThreads.forEach(ExecutorService::shutdownNow); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - //TODO: cleanup repositories that we no longer manage in this node. + for (TenantId tenantId : vcService.getActiveRepositoryTenants()) { + if (!partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId).isMyPartition()) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + pendingCommitMap.remove(tenantId); + vcService.clearRepository(tenantId); + } catch (Exception e) { + log.warn("[{}] Failed to cleanup the tenant repository", tenantId, e); + } finally { + lock.unlock(); + } + } + } consumer.subscribe(event.getPartitions()); } @@ -143,6 +178,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe void consumerLoop(TbQueueConsumer> consumer) { while (!stopped && !consumer.isStopped()) { + List> futures = new ArrayList<>(); try { List> msgs = consumer.poll(pollDuration); if (msgs.isEmpty()) { @@ -151,46 +187,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe for (TbProtoQueueMsg msgWrapper : msgs) { ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); - var lock = getRepoLock(ctx.getTenantId()); - lock.lock(); - try { - if (msg.hasClearRepositoryRequest()) { - handleClearRepositoryCommand(ctx); - } else { - if (msg.hasTestRepositoryRequest()) { - handleTestRepositoryCommand(ctx); - } else if (msg.hasInitRepositoryRequest()) { - handleInitRepositoryCommand(ctx); - } else { - var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); - var newSettings = ctx.getSettings(); - if (!newSettings.equals(currentSettings)) { - vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); - } else { - vcService.fetch(ctx.getTenantId()); - } - if (msg.hasCommitRequest()) { - handleCommitRequest(ctx, msg.getCommitRequest()); - } else if (msg.hasListBranchesRequest()) { - handleListBranches(ctx, msg.getListBranchesRequest()); - } else if (msg.hasListEntitiesRequest()) { - handleListEntities(ctx, msg.getListEntitiesRequest()); - } else if (msg.hasListVersionRequest()) { - handleListVersions(ctx, msg.getListVersionRequest()); - } else if (msg.hasEntityContentRequest()) { - handleEntityContentRequest(ctx, msg.getEntityContentRequest()); - } else if (msg.hasEntitiesContentRequest()) { - handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); - } - } - } - } catch (Exception e) { - reply(ctx, Optional.of(e)); - } finally { - lock.unlock(); - } + long startTs = System.currentTimeMillis(); + log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); + ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + logTaskExecution(ctx, future, startTs); + futures.add(future); + } + try { + Futures.allAsList(futures).get(packProcessingTimeout, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Timeout for processing the version control tasks.", e); } - //TODO: handle timeouts and async processing for multiple tenants; consumer.commit(); } catch (Exception e) { if (!stopped) { @@ -206,6 +213,51 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } + private Void processMessage(VersionControlRequestCtx ctx, ToVersionControlServiceMsg msg) { + var lock = getRepoLock(ctx.getTenantId()); + lock.lock(); + try { + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(ctx); + } else { + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } else { + var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); + var newSettings = ctx.getSettings(); + if (!newSettings.equals(currentSettings)) { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } + if (msg.hasCommitRequest()) { + handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } + } + } + } catch (Exception e) { + reply(ctx, Optional.of(e)); + } finally { + lock.unlock(); + } + return null; + } + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { var entityType = EntityType.valueOf(request.getEntityType()); String path = getRelativePath(entityType, null); @@ -273,6 +325,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); if (request.hasPrepareMsg()) { + vcService.fetch(ctx.getTenantId()); prepareCommit(ctx, txId, request.getPrepareMsg()); } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); @@ -414,4 +467,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } + + private void logTaskExecution(VersionControlRequestCtx ctx, ListenableFuture future, long startTs) { + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback() { + + @Override + public void onSuccess(@Nullable Object result) { + log.trace("[{}][{}] Task processing took: {}ms", ctx.getTenantId(), ctx.getRequestId(), (System.currentTimeMillis() - startTs)); + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}][{}] Task failed: ", ctx.getTenantId(), ctx.getRequestId(), t); + } + }, MoreExecutors.directExecutor()); + } + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..e30dd8aa20 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -35,19 +35,17 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -61,10 +59,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") private String repositoriesFolder; - @Value("${vc.git.repos-poll-interval:60}") - private long reposPollInterval; - - private ScheduledExecutorService scheduler; private final Map repositories = new ConcurrentHashMap<>(); @PostConstruct @@ -72,24 +66,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { if (StringUtils.isEmpty(repositoriesFolder)) { repositoriesFolder = defaultFolder; } - scheduler = Executors.newSingleThreadScheduledExecutor(); - scheduler.scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); } - @PreDestroy - public void stop() { - if (scheduler != null) { - scheduler.shutdownNow(); - } + @Override + public Set getActiveRepositoryTenants() { + return new HashSet<>(repositories.keySet()); } @Override @@ -151,6 +132,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @SneakyThrows @Override public void cleanUp(PendingCommit commit) { + log.debug("[{}] Cleanup tenant repository started.", commit.getTenantId()); GitRepository repository = checkRepository(commit.getTenantId()); try { repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); @@ -161,6 +143,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } repository.resetAndClean(); repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + log.debug("[{}] Cleanup tenant repository completed.", commit.getTenantId()); } @Override @@ -172,7 +155,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void fetch(TenantId tenantId) throws GitAPIException { var repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Fetching tenant repository.", tenantId); repository.fetch(); + log.debug("[{}] Fetched tenant repository.", tenantId); } } @@ -232,6 +217,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Override public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { clearRepository(tenantId); + log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; if (Files.exists(repositoryDirectory)) { @@ -241,6 +227,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings, repositoryDirectory.toFile()); repositories.put(tenantId, repository); + log.debug("[{}] Init tenant repository completed.", tenantId); } @Override @@ -253,8 +240,10 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Clear tenant repository started.", tenantId); FileUtils.deleteDirectory(new File(repository.getDirectory())); repositories.remove(tenantId); + log.debug("[{}] Clear tenant repository completed.", tenantId); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 075e08a33e..9d57eee3ec 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -26,9 +26,12 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.io.IOException; import java.util.List; +import java.util.Set; public interface GitRepositoryService { + Set getActiveRepositoryTenants(); + void prepareCommit(PendingCommit pendingCommit); PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; diff --git a/msa/pom.xml b/msa/pom.xml index 888291502b..dd40348bc8 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -41,6 +41,7 @@ tb vc-executor + vc-executor-docker js-executor web-ui tb-node diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile new file mode 100644 index 0000000000..c2f9640a6b --- /dev/null +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk11 + +COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin + +RUN yes | dpkg -i /tmp/${pkg.name}.deb +RUN rm /tmp/${pkg.name}.deb + +RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : + +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-vc-executor.sh"] diff --git a/msa/vc-executor-docker/docker/start-tb-vc-executor.sh b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh new file mode 100755 index 0000000000..4384008fd3 --- /dev/null +++ b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml new file mode 100644 index 0000000000..39fc35f32a --- /dev/null +++ b/msa/vc-executor-docker/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor-docker + pom + + ThingsBoard Version Control Executor Microservice + https://thingsboard.io + ThingsBoard Version Control Executor Microservice + + + UTF-8 + ${basedir}/../.. + tb-vc-executor + tb-vc-executor + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard.msa + vc-executor + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-vc-executor-deb + package + + copy + + + + + org.thingsboard.msa + vc-executor + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml index 7e4df36117..0125354371 100644 --- a/msa/vc-executor/pom.xml +++ b/msa/vc-executor/pom.xml @@ -38,7 +38,7 @@ false process-resources package - tb-mqtt-transport + tb-vc-executor false ${project.build.directory}/windows ThingsBoard Version Control Executor Service @@ -46,6 +46,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common version-control diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index d30818110d..6978f94b51 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -15,17 +15,17 @@ package org.thingsboard.server.vc; /** */ import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; import java.util.Arrays; -@SpringBootConfiguration +@SpringBootApplication @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) +@ComponentScan({"org.thingsboard.server", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java new file mode 100644 index 0000000000..0f4ac69948 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class VersionControlQueueRoutingInfoService implements QueueRoutingInfoService { + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java new file mode 100644 index 0000000000..ebd7ed07b3 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class VersionControlTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + //This dummy implementation is ok since Version Control service does not produce any rule engine messages. + return new TenantRoutingInfo(tenantId, false, false); + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 2834934ee0..572d093dde 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -25,6 +25,7 @@ + diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 1de47e2750..8c34a91e80 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -23,5 +23,163 @@ server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). - port: "${HTTP_BIND_PORT:8080}" + port: "${HTTP_BIND_PORT:8086}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +queue: + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + - key: max.poll.records + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + ota: + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +service: + type: "${TB_SERVICE_TYPE:tb-vc-executor}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" \ No newline at end of file From dbec05a2c173ec1446170c2300b757b1e0d05819 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 10:39:48 +0300 Subject: [PATCH 082/178] Test fix --- .../service/cluster/routing/HashPartitionServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java index 74738c361f..6aafa833b1 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java @@ -71,6 +71,8 @@ public class HashPartitionServiceTest { queueRoutingInfoService); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10); + ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); + ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() .setServiceId("tb-core-0") From ca3c95afca1e55290e82cbe4e1a213402c4c9a57 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 12:26:23 +0300 Subject: [PATCH 083/178] UI: Implement entitiy versions table --- .../EntitiesVersionControlController.java | 7 +- .../DefaultGitVersionControlQueueService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 3 +- ui-ngx/src/app/core/auth/auth.actions.ts | 11 +- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 6 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 5 + ui-ngx/src/app/core/auth/auth.service.ts | 12 +- .../http/entities-version-control.service.ts | 27 +- ui-ngx/src/app/core/services/menu.service.ts | 17 ++ .../home/components/home-components.module.ts | 13 +- .../vc/entity-versions-table.component.html | 81 ++++++ .../vc/entity-versions-table.component.scss | 100 +++++++ .../vc/entity-versions-table.component.ts | 244 ++++++++++++++++++ .../version-control-settings.component.html | 2 +- .../version-control-settings.component.scss | 3 + .../vc}/version-control-settings.component.ts | 53 ++-- .../vc/version-control.component.html | 25 ++ .../vc/version-control.component.scss | 18 ++ .../vc/version-control.component.ts | 61 +++++ .../home/pages/admin/admin-routing.module.ts | 4 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ...sion-control-admin-settings.component.html | 18 ++ ...ersion-control-admin-settings.component.ts | 44 ++++ .../pages/device/device-tabs.component.html | 5 + .../modules/home/pages/home-pages.module.ts | 4 +- .../home/pages/vc/vc-routing.module.ts | 44 ++++ .../app/modules/home/pages/vc/vc.module.ts | 31 +++ .../vc/branch-autocomplete.component.html | 5 +- .../vc/branch-autocomplete.component.ts | 147 ++++++++--- ui-ngx/src/app/shared/models/asset.models.ts | 4 +- ui-ngx/src/app/shared/models/base-data.ts | 6 + .../src/app/shared/models/customer.model.ts | 3 +- .../src/app/shared/models/dashboard.models.ts | 4 +- ui-ngx/src/app/shared/models/device.models.ts | 6 +- .../app/shared/models/rule-chain.models.ts | 4 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 10 +- 38 files changed, 948 insertions(+), 87 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.html (98%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.scss (95%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.ts (82%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc.module.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0efa52dee2..bafc93c19f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -294,11 +294,14 @@ public class EntitiesVersionControlController extends BaseController { String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + remoteBranches.forEach(branch -> { + if (!branch.equals(defaultBranch)) { + infos.add(new BranchInfo(branch, false)); + } + }); return infos; }, MoreExecutors.directExecutor())); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fbee0502e2..f738582bb3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -411,7 +411,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } if (vcSettings != null) { builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings))); - } else { + } else if (request.requiresSettings()) { throw new RuntimeException("No entity version control settings provisioned!"); } return builder; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 6208b913f4..20ed03795d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -170,7 +170,7 @@ public class GitRepository { public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); + return new PageData<>(); } LogCommand command = git.log() .add(branchId) @@ -313,6 +313,7 @@ public class GitRepository { Function mapper, PageLink pageLink, Function> comparatorFunction) { + iterable = Streams.stream(iterable).collect(Collectors.toList()); int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 6edbdcd5a2..872607b822 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -23,7 +23,8 @@ export enum AuthActionTypes { UNAUTHENTICATED = '[Auth] Unauthenticated', LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', - UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id' + UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', + UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' } export class ActionAuthAuthenticated implements Action { @@ -54,5 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } +export class ActionAuthUpdateHasVersionControl implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; + + constructor(readonly payload: { hasVersionControl: boolean }) {} +} + export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 9267d48e28..a69b783526 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,6 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; + hasVersionControl: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index be2712300d..9d9404524b 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -23,7 +23,8 @@ const emptyUserAuthState: AuthPayload = { userTokenAccessEnabled: false, forceFullscreen: false, allowedDashboardIds: [], - edgesSupportEnabled: false + edgesSupportEnabled: false, + hasVersionControl: false }; export const initialState: AuthState = { @@ -54,6 +55,9 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; + case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + return { ...state, ...action.payload}; + default: return state; } diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index aaadf00ec3..7a099d5acd 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,6 +55,11 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); +export const selectHasVersionControl = createSelector( + selectAuthState, + (state: AuthState) => state.hasVersionControl +); + export function getCurrentAuthState(store: Store): AuthState { let state: AuthState; store.pipe(select(selectAuth), take(1)).subscribe( diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6b129da6ad..6fe809955f 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,17 +437,27 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } + private loadHasVersionControl(authUser: AuthUser): Observable { + if (authUser.authority === Authority.TENANT_ADMIN) { + return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + } else { + return of(false); + } + } + private loadSystemParams(authPayload: AuthPayload): Observable { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), + this.loadHasVersionControl(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled}; + const hasVersionControl: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 231581a02f..f8668f2ecf 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,12 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { Observable } from 'rxjs'; -import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { PageData } from '@shared/models/page/page-data'; +import { DeviceInfo } from '@shared/models/device.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; @Injectable({ providedIn: 'root' @@ -37,4 +42,24 @@ export class EntitiesVersionControlService { public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); } + + public listEntityVersions(pageLink: PageLink, branch: string, + externalEntityId: EntityId, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listEntityTypeVersions(pageLink: PageLink, branch: string, + entityType: EntityType, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listVersions(pageLink: PageLink, branch: string, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 56191ec948..ee81f007ee 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -343,6 +343,13 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, + { + id: guid(), + name: 'version-control.version-control', + type: 'link', + path: '/vc', + icon: 'history' + }, { id: guid(), name: 'audit-log.audit-logs', @@ -492,6 +499,16 @@ export class MenuService { } ] }, + { + name: 'version-control.management', + places: [ + { + name: 'version-control.version-control', + icon: 'history', + path: '/vc' + } + ] + }, { name: 'audit-log.audit', places: [ diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 05067e78ee..49f1f57fd1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -154,6 +154,9 @@ import { QueueFormComponent } from '@home/components/queue/queue-form.component' import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; +import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; @NgModule({ declarations: @@ -278,7 +281,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], imports: [ CommonModule, @@ -397,7 +403,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html new file mode 100644 index 0000000000..b47c2fcc79 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -0,0 +1,81 @@ + +
+
+ +
+
+ {{(singleEntityMode ? 'version-control.entity-versions' : 'version-control.versions') | translate}} + + +
+ + +
+ +
+ + + {{ 'version-control.created-time' | translate }} + + {{ entityVersion.timestamp | date:'yyyy-MM-dd HH:mm:ss' }} + + + + {{ 'version-control.version-id' | translate }} + + {{ entityVersion.id }} + + + + {{ 'version-control.version-name' | translate }} + + {{ entityVersion.name }} + + + + +
+ {{ + singleEntityMode + ? 'version-control.no-entity-versions-text' + : 'version-control.no-versions-text' + }} +
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss new file mode 100644 index 0000000000..9017de60b8 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + width: 100%; + height: 100%; + display: block; + .tb-entity-table { + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; + + .mat-toolbar-tools{ + min-height: auto; + } + + .title-container{ + overflow: hidden; + } + + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .table-container { + overflow: auto; + } + + .tb-entity-table-info{ + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .button-widget-action{ + margin-left: auto; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + @media #{$mat-xs} { + .mat-toolbar { + height: auto; + min-height: 100px; + + .tb-entity-table-title{ + padding-bottom: 5px; + width: 100%; + } + } + } +} + +:host ::ng-deep { + .mat-sort-header-sorted .mat-sort-header-arrow { + opacity: 1 !important; + } + tb-branch-autocomplete { + mat-form-field { + font-size: 16px; + width: 200px; + + .mat-form-field-wrapper { + padding-bottom: 0; + } + + .mat-form-field-underline { + bottom: 0; + } + + @media #{$mat-xs} { + width: 100%; + + .mat-form-field-infix { + width: auto !important; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts new file mode 100644 index 0000000000..4fc33beafb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -0,0 +1,244 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + Input, + OnDestroy, + OnInit, + ViewChild +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityId } from '@shared/models/id/entity-id'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, map, tap } from 'rxjs/operators'; +import { EntityVersion } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { ResizeObserver } from '@juggle/resize-observer'; +import { hidePageSizePixelValue } from '@shared/models/constants'; +import { Direction, SortOrder } from '@shared/models/page/sort-order'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; + +@Component({ + selector: 'tb-entity-versions-table', + templateUrl: './entity-versions-table.component.html', + styleUrls: ['./entity-versions-table.component.scss'] +}) +export class EntityVersionsTableComponent extends PageComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('branchAutocompleteComponent') branchAutocompleteComponent: BranchAutocompleteComponent; + + @Input() + singleEntityMode = false; + + displayedColumns = ['timestamp', 'id', 'name']; + pageLink: PageLink; + dataSource: EntityVersionsDatasource; + hidePageSize = false; + + branch: string = null; + + activeValue = false; + dirtyValue = false; + externalEntityIdValue: EntityId; + + viewsInited = false; + + private componentResize$: ResizeObserver; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + if (this.viewsInited) { + this.initFromDefaultBranch(); + } + } + } + } + + @Input() + set externalEntityId(externalEntityId: EntityId) { + if (this.externalEntityIdValue !== externalEntityId) { + this.externalEntityIdValue = externalEntityId; + this.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.dirtyValue = true; + } + } + } + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private elementRef: ElementRef) { + super(store); + this.dirtyValue = !this.activeValue; + const sortOrder: SortOrder = { property: 'timestamp', direction: Direction.DESC }; + this.pageLink = new PageLink(10, 0, null, sortOrder); + this.dataSource = new EntityVersionsDatasource(this.entitiesVersionControlService); + } + + ngOnInit() { + this.componentResize$ = new ResizeObserver(() => { + const showHidePageSize = this.elementRef.nativeElement.offsetWidth < hidePageSizePixelValue; + if (showHidePageSize !== this.hidePageSize) { + this.hidePageSize = showHidePageSize; + this.cd.markForCheck(); + } + }); + this.componentResize$.observe(this.elementRef.nativeElement); + } + + ngOnDestroy() { + if (this.componentResize$) { + this.componentResize$.disconnect(); + } + } + + branchChanged(newBranch: string) { + this.branch = newBranch; + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } + + ngAfterViewInit() { + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); + merge(this.sort.sortChange, this.paginator.page) + .pipe( + tap(() => this.updateData()) + ) + .subscribe(); + this.viewsInited = true; + if (!this.singleEntityMode) { + this.initFromDefaultBranch(); + } + } + + vcExport($event: Event) { + if ($event) { + $event.stopPropagation(); + } + } + + private initFromDefaultBranch() { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + } + + private updateData() { + this.pageLink.page = this.paginator.pageIndex; + this.pageLink.pageSize = this.paginator.pageSize; + this.pageLink.sortOrder.property = this.sort.active; + this.pageLink.sortOrder.direction = Direction[this.sort.direction.toUpperCase()]; + this.dataSource.loadEntityVersions(this.singleEntityMode, this.branch, this.externalEntityIdValue, this.pageLink); + } + + private resetSortAndFilter(update: boolean) { + this.branch = null; + this.pageLink.textSearch = null; + if (this.viewsInited) { + this.paginator.pageIndex = 0; + const sortable = this.sort.sortables.get('timestamp'); + this.sort.active = sortable.id; + this.sort.direction = 'desc'; + if (update) { + this.initFromDefaultBranch(); + } + } + } +} + +class EntityVersionsDatasource implements DataSource { + + private entityVersionsSubject = new BehaviorSubject([]); + private pageDataSubject = new BehaviorSubject>(emptyPageData()); + + public pageData$ = this.pageDataSubject.asObservable(); + + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} + + connect(collectionViewer: CollectionViewer): Observable> { + return this.entityVersionsSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entityVersionsSubject.complete(); + this.pageDataSubject.complete(); + } + + loadEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + const result = new ReplaySubject>(); + this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entityVersionsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + } + ); + return result; + } + + fetchEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + if (!branch) { + return of(emptyPageData()); + } else { + if (singleEntityMode) { + if (externalEntityId) { + return this.entitiesVersionControlService.listEntityVersions(pageLink, branch, externalEntityId, {ignoreErrors: true}); + } else { + return of(emptyPageData()); + } + } else { + return this.entitiesVersionControlService.listVersions(pageLink, branch, {ignoreErrors: true}); + } + } + } + + isEmpty(): Observable { + return this.entityVersionsSubject.pipe( + map((entityVersions) => !entityVersions.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html similarity index 98% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html index b512bb96f5..b48162d06f 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.git-repository-settings diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss similarity index 95% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss index ede3570e68..d1d55faf19 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss @@ -14,6 +14,9 @@ * limitations under the License. */ :host { + mat-card.vc-settings { + margin: 8px; + } .fields-group { padding: 0 16px 8px; margin-bottom: 10px; diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts similarity index 82% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index c2626ec062..2970987be7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { AdminService } from '@core/http/admin.service'; import { @@ -30,13 +30,21 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { TranslateService } from '@ngx-translate/core'; import { isNotEmptyStr } from '@core/utils'; import { DialogService } from '@core/services/dialog.service'; +import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; +import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; @Component({ selector: 'tb-version-control-settings', templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] + styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] }) -export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class VersionControlSettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; versionControlSettingsForm: FormGroup; settings: EntitiesVersionControlSettings = null; @@ -62,7 +70,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On ngOnInit() { this.versionControlSettingsForm = this.fb.group({ repositoryUri: [null, [Validators.required]], - defaultBranch: [null, []], + defaultBranch: ['main', []], authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], username: [null, []], password: [null, []], @@ -77,16 +85,29 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { this.updateValidators(false); }); - this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + this.store.pipe( + select(selectHasVersionControl), + mergeMap((hasVersionControl) => { + if (hasVersionControl) { + return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( (settings) => { this.settings = settings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; + if (this.settings != null) { + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); }); } @@ -112,6 +133,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On } this.versionControlSettingsForm.reset(this.settings); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); } ); } @@ -131,18 +153,15 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.showChangePrivateKeyPassword = false; this.changePrivateKeyPassword = false; formDirective.resetForm(); - this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); } ); } }); } - confirmForm(): FormGroup { - return this.versionControlSettingsForm; - } - changePasswordChanged() { if (this.changePassword) { this.versionControlSettingsForm.get('password').patchValue(''); diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html new file mode 100644 index 0000000000..903460054b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -0,0 +1,25 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss new file mode 100644 index 0000000000..da8df4b469 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts new file mode 100644 index 0000000000..07bdbcba25 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -0,0 +1,61 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { FormGroup } from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-version-control', + templateUrl: './version-control.component.html', + styleUrls: ['./version-control.component.scss'] +}) +export class VersionControlComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent', {static: false}) versionControlSettingsComponent: VersionControlSettingsComponent; + + @Input() + detailsMode = false; + + @Input() + active = true; + + @Input() + singleEntityMode = false; + + @Input() + externalEntityId: EntityId; + + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + + constructor(private store: Store) { + + } + + ngOnInit() { + + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } + +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index d95b55e5f4..958e7b1c9b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -226,7 +226,7 @@ const routes: Routes = [ }, { path: 'vc', - component: VersionControlSettingsComponent, + component: VersionControlAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index aab8d4d56f..51612270df 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlSettingsComponent } from '@home/pages/admin/version-contr HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlSettingsComponent + VersionControlAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html new file mode 100644 index 0000000000..869067e1fb --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html @@ -0,0 +1,18 @@ + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts new file mode 100644 index 0000000000..a203f64426 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; + +@Component({ + selector: 'tb-version-control-admin-settings', + templateUrl: './version-control-admin-settings.component.html', + styleUrls: [] +}) +export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 3bec9b906c..33aac9ba44 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index 576ab1c9b2..feea05d76a 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -36,6 +36,7 @@ import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; +import { VcModule } from '@home/pages/vc/vc.module'; @NgModule({ exports: [ @@ -56,7 +57,8 @@ import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; AuditLogModule, ApiUsageModule, OtaUpdateModule, - UserModule + UserModule, + VcModule ], providers: [ { diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts new file mode 100644 index 0000000000..6054aaa14d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { ConfirmOnExitGuard } from '@core/guards/confirm-on-exit.guard'; +import { Authority } from '@shared/models/authority.enum'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; + +const routes: Routes = [ + { + path: 'vc', + component: VersionControlComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'version-control.version-control', + breadcrumb: { + label: 'version-control.version-control', + icon: 'history' + } + } + } +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) +export class VcRoutingModule { } diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts new file mode 100644 index 0000000000..4944356a9d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts @@ -0,0 +1,31 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { VcRoutingModule } from '@home/pages/vc/vc-routing.module'; + +@NgModule({ + declarations: [ + ], + imports: [ + CommonModule, + SharedModule, + VcRoutingModule + ] +}) +export class VcModule { } diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index 3a37d707ea..b189d70199 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - + {{ 'version-control.branch' | translate }} - + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts index 421710181a..e72235d89e 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -14,7 +14,17 @@ /// limitations under the License. /// -import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + forwardRef, + Input, + NgZone, + OnInit, + ViewChild +} from '@angular/core'; import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; import { Observable, of } from 'rxjs'; import { @@ -24,6 +34,7 @@ import { map, publishReplay, refCount, + share, switchMap, tap } from 'rxjs/operators'; @@ -32,6 +43,7 @@ import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { BranchInfo } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { isNotEmptyStr } from '@core/utils'; @Component({ selector: 'tb-branch-autocomplete', @@ -60,27 +72,48 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.requiredValue = coerceBooleanProperty(value); } + private disabledValue: boolean; + + get disabled(): boolean { + return this.disabledValue; + } + @Input() - disabled: boolean; + set disabled(value: boolean) { + this.disabledValue = coerceBooleanProperty(value); + if (this.disabledValue) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } @Input() selectDefaultBranch = true; + @Input() + selectionMode = false; + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; - filteredBranches: Observable>; + filteredBranches: Observable>; - branches: Observable>; + branches: Observable> = null; + + defaultBranch: BranchInfo = null; searchText = ''; private dirty = false; + private ignoreClosedPanel = false; + private propagateChange = (v: any) => { }; constructor(private store: Store, private entitiesVersionControlService: EntitiesVersionControlService, - private fb: FormBuilder) { + private fb: FormBuilder, + private zone: NgZone) { this.branchFormGroup = this.fb.group({ branch: [null, []] }); @@ -94,17 +127,36 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } ngOnInit() { - - this.branches = null; this.filteredBranches = this.branchFormGroup.get('branch').valueChanges .pipe( + tap((value: BranchInfo | string) => { + let modelValue: BranchInfo | null; + if (typeof value === 'string' || !value) { + if (!this.selectionMode && typeof value === 'string' && isNotEmptyStr(value)) { + modelValue = {name: value, default: false}; + } else { + modelValue = null; + } + } else { + modelValue = value; + } + this.updateView(modelValue); + }), + map(value => { + if (value) { + if (typeof value === 'string') { + return value; + } else { + return value.name; + } + } else { + return ''; + } + }), debounceTime(150), distinctUntilChanged(), - tap(value => { - this.updateView(value); - }), - map(value => value ? value : ''), - switchMap(branch => this.fetchBranches(branch)) + switchMap(name => this.fetchBranches(name)), + share() ); } @@ -113,24 +165,16 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; - if (this.disabled) { - this.branchFormGroup.disable({emitEvent: false}); - } else { - this.branchFormGroup.enable({emitEvent: false}); - } } - selectDefaultBranchIfNeeded(): void { - if (this.selectDefaultBranch && !this.modelValue) { - this.getBranches().subscribe( + selectDefaultBranchIfNeeded(ignoreLoading = true, force = false): void { + if ((this.selectDefaultBranch && !this.modelValue) || force) { + this.getBranches(ignoreLoading).subscribe( (data) => { - if (data && data.length) { - const defaultBranch = data.find(branch => branch.default); - if (defaultBranch) { - this.modelValue = defaultBranch.name; - this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); - this.propagateChange(this.modelValue); - } + if (this.defaultBranch || force) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); } } ); @@ -141,9 +185,9 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.searchText = ''; this.modelValue = value; if (value != null) { - this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue({name: value}, {emitEvent: false}); } else { - this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: false}); this.selectDefaultBranchIfNeeded(); } this.dirty = true; @@ -156,31 +200,53 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } } - updateView(value: string | null) { - if (this.modelValue !== value) { - this.modelValue = value; + onPanelClosed() { + if (this.ignoreClosedPanel) { + this.ignoreClosedPanel = false; + } else { + if (this.selectionMode && !this.branchFormGroup.get('branch').value && this.defaultBranch) { + this.zone.run(() => { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: true}); + }, 0); + } + } + } + + updateView(value: BranchInfo | null) { + if (this.modelValue !== value?.name) { + this.modelValue = value?.name; this.propagateChange(this.modelValue); } } - displayBranchFn(branch?: string): string | undefined { - return branch ? branch : undefined; + displayBranchFn(branch?: BranchInfo): string | undefined { + return branch ? branch.name : undefined; } - fetchBranches(searchText?: string): Observable> { + fetchBranches(searchText?: string): Observable> { this.searchText = searchText; return this.getBranches().pipe( - map(branches => branches.map(branch => branch.name).filter(branchName => { - return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; - })) + map(branches => { + let res = branches.filter(branch => { + return searchText ? branch.name.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (!this.selectionMode && isNotEmptyStr(searchText) && !res.find(b => b.name === searchText)) { + res = [{name: searchText, default: false}, ...res]; + } + return res; + } + ) ); } - getBranches(): Observable> { + getBranches(ignoreLoading = true): Observable> { if (!this.branches) { - const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading, ignoreErrors: true}); this.branches = branchesObservable.pipe( catchError(() => of([] as Array)), + tap((data) => { + this.defaultBranch = data.find(branch => branch.default); + }), publishReplay(1), refCount() ); @@ -189,6 +255,7 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } clear() { + this.ignoreClosedPanel = true; this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); setTimeout(() => { this.branchInput.nativeElement.blur(); diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index d38aee4d96..115eea4610 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { AssetId } from './id/asset-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; -export interface Asset extends BaseData { +export interface Asset extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/base-data.ts b/ui-ngx/src/app/shared/models/base-data.ts index 3af9bb9059..7b3ea9a642 100644 --- a/ui-ngx/src/app/shared/models/base-data.ts +++ b/ui-ngx/src/app/shared/models/base-data.ts @@ -27,6 +27,12 @@ export interface BaseData { label?: string; } +export interface ExportableEntity { + createdTime?: number; + id?: T; + externalId?: T; +} + export function hasIdEquals(id1: HasId, id2: HasId): boolean { if (isDefinedAndNotNull(id1) && isDefinedAndNotNull(id2)) { return id1.id === id2.id; diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 435edf263f..67c1429189 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -17,8 +17,9 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; import { TenantId } from './id/tenant-id'; +import { ExportableEntity } from '@shared/models/base-data'; -export interface Customer extends ContactBased { +export interface Customer extends ContactBased, ExportableEntity { tenantId: TenantId; title: string; additionalInfo?: any; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 4332e484f6..12e1c83ada 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DashboardId } from '@shared/models/id/dashboard-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; @@ -23,7 +23,7 @@ import { Timewindow } from '@shared/models/time/time.models'; import { EntityAliases } from './alias.models'; import { Filters } from '@shared/models/query/query.models'; -export interface DashboardInfo extends BaseData { +export interface DashboardInfo extends BaseData, ExportableEntity { tenantId?: TenantId; title?: string; image?: string; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index f5fc25e69d..c4c29f5da6 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DeviceId } from './id/device-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; @@ -560,7 +560,7 @@ export interface DeviceProfileData { provisionConfiguration?: DeviceProvisionConfiguration; } -export interface DeviceProfile extends BaseData { +export interface DeviceProfile extends BaseData, ExportableEntity { tenantId?: TenantId; name: string; description?: string; @@ -685,7 +685,7 @@ export interface DeviceData { transportConfiguration: DeviceTransportConfiguration; } -export interface Device extends BaseData { +export interface Device extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/rule-chain.models.ts b/ui-ngx/src/app/shared/models/rule-chain.models.ts index 85d6047191..01f9d268c3 100644 --- a/ui-ngx/src/app/shared/models/rule-chain.models.ts +++ b/ui-ngx/src/app/shared/models/rule-chain.models.ts @@ -14,14 +14,14 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainId } from '@shared/models/id/rule-chain-id'; import { RuleNodeId } from '@shared/models/id/rule-node-id'; import { RuleNode, RuleNodeComponentDescriptor, RuleNodeType } from '@shared/models/rule-node.models'; import { ComponentType } from '@shared/models/component-descriptor.models'; -export interface RuleChain extends BaseData { +export interface RuleChain extends BaseData, ExportableEntity { tenantId: TenantId; name: string; firstRuleNodeId: RuleNodeId; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0c890ff9b4..d574a1b55f 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -43,6 +43,7 @@ export interface BranchInfo { } export interface EntityVersion { + timestamp: number; id: string; name: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c37ba685e0..d5bac68d48 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3109,6 +3109,8 @@ "json-value-required": "JSON value is required." }, "version-control": { + "version-control": "Version control", + "management": "Version control management", "branch": "Branch", "select-branch": "Select branch", "branch-required": "Branch is required", @@ -3118,7 +3120,13 @@ "version-name-required": "Version name is required", "export-entity-relations": "Export entity relations", "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", - "export-to-git": "Export to Git" + "export-to-git": "Export to Git", + "entity-versions": "Entity versions", + "versions": "Versions", + "created-time": "Created time", + "version-id": "Version ID", + "no-entity-versions-text": "No entity versions found", + "no-versions-text": "No versions found" }, "widget": { "widget-library": "Widgets Library", From 4f2e1a25fdd772ed27dba9509dbec639dc46bd60 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 12:58:55 +0300 Subject: [PATCH 084/178] Docker compose scripts for TB Version Control Executor --- .../vc/EntitiesVersionControlService.java | 2 - .../queue/discovery/HashPartitionService.java | 2 +- .../DefaultClusterVersionControlService.java | 17 ++++--- docker/.env | 1 + docker/.gitignore | 1 + docker/docker-compose.confluent.yml | 6 +++ docker/docker-compose.kafka.yml | 10 ++++ docker/docker-compose.postgres.volumes.yml | 10 ++++ docker/docker-compose.pubsub.yml | 33 +++--------- docker/docker-compose.rabbitmq.yml | 32 +++--------- docker/docker-compose.service-bus.yml | 30 +++-------- docker/docker-compose.yml | 32 ++++++++++++ docker/docker-create-log-folders.sh | 2 + docker/tb-vc-executor.env | 2 + docker/tb-vc-executor/conf/logback.xml | 51 +++++++++++++++++++ .../tb-vc-executor/conf/tb-vc-executor.conf | 23 +++++++++ .../server/msa/ThingsBoardDbInstaller.java | 10 +++- msa/vc-executor-docker/docker/Dockerfile | 3 ++ 18 files changed, 181 insertions(+), 86 deletions(-) create mode 100644 docker/tb-vc-executor.env create mode 100644 docker/tb-vc-executor/conf/logback.xml create mode 100644 docker/tb-vc-executor/conf/tb-vc-executor.conf diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 566e093cb4..08b92db96a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -50,10 +50,8 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index e2325be878..583e196e2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,7 +54,7 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; - @Value("${queue.vc.topic}") + @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") private Integer vcPartitions; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..5fcb9985bd 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -188,8 +188,9 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); long startTs = System.currentTimeMillis(); - log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); - ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + log.trace("[{}][{}] RECEIVED task: {}", ctx.getTenantId(), ctx.getRequestId(), msg); + int threadIdx = Math.abs(ctx.getTenantId().hashCode() % ioPoolSize); + ListenableFuture future = ioThreads.get(threadIdx).submit(() -> processMessage(ctx, msg)); logTaskExecution(ctx, future, startTs); futures.add(future); } @@ -435,13 +436,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); if (e.isPresent()) { + log.debug("[{}][{}] Failed to process task", ctx.getTenantId(), ctx.getRequestId(), e.get()); builder.setError(e.get().getMessage()); - } - if (enrichFunction != null) { - builder = enrichFunction.apply(builder); } else { - builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + if (enrichFunction != null) { + builder = enrichFunction.apply(builder); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + log.debug("[{}][{}] Processed task", ctx.getTenantId(), ctx.getRequestId()); } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); diff --git a/docker/.env b/docker/.env index 11e44fdde8..af0f537603 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport +TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor TB_VERSION=latest diff --git a/docker/.gitignore b/docker/.gitignore index eee422d573..9c4c778f28 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -5,4 +5,5 @@ tb-node/db/** tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log +tb-vc-executor/log/** !.env diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 6983dd3d3e..3d5abd0abe 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -61,3 +61,9 @@ services: tb-snmp-transport: env_file: - queue-confluent.env + tb-vc-executor1: + env_file: + - queue-confluent.env + tb-vc-executor2: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 2184528e53..09c4554562 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -90,3 +90,13 @@ services: - queue-kafka.env depends_on: - kafka + tb-vc-executor1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-vc-executor2: + env_file: + - queue-kafka.env + depends_on: + - kafka \ No newline at end of file diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 704e9400ae..019e087c48 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -53,6 +53,13 @@ services: tb-snmp-transport: volumes: - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + volumes: postgres-db-volume: @@ -76,3 +83,6 @@ volumes: tb-snmp-transport-log-volume: external: true name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 0364957ee6..c03132d730 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -23,59 +23,40 @@ services: tb-core1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-pubsub.env + tb-vc-executor2: + env_file: + - queue-pubsub.env + diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index 1eb37709e5..d1acc32014 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-rabbitmq.env + tb-vc-executor2: + env_file: + - queue-rabbitmq.env \ No newline at end of file diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index c511658b31..6e39de0baa 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -23,57 +23,39 @@ services: tb-core1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-service-bus.env tb-lwm2m-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-service-bus.env + tb-vc-executor2: + env_file: + - queue-service-bus.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82fbaeed74..1ba6eda32d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -257,6 +257,38 @@ services: - "8080" env_file: - tb-web-ui.env + tb-vc-executor1: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor1 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 + tb-vc-executor2: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor2 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 5af0f96377..ba945a19df 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -26,3 +26,5 @@ mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log + +mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log diff --git a/docker/tb-vc-executor.env b/docker/tb-vc-executor.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-vc-executor.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml new file mode 100644 index 0000000000..dc95b3a885 --- /dev/null +++ b/docker/tb-vc-executor/conf/logback.xml @@ -0,0 +1,51 @@ + + + + + + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.log + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/docker/tb-vc-executor/conf/tb-vc-executor.conf b/docker/tb-vc-executor/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..f140e3fb76 --- /dev/null +++ b/docker/tb-vc-executor/conf/tb-vc-executor.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-vc-executor/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-vc-executor/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-vc-executor.out +export LOADER_PATH=/usr/share/tb-vc-executor/conf diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 417799f64f..de55a3afa3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -34,6 +34,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; + private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; private final DockerComposeExecutor dockerCompose; @@ -44,6 +45,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; + private final String tbVcExecutorLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -61,6 +63,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; + tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -72,6 +75,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); + env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); dockerCompose.withEnv(env); } @@ -104,6 +108,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); + dockerCompose.invokeDocker(); + dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); @@ -126,10 +133,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); + copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); } diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile index c2f9640a6b..f4ff25e0e5 100644 --- a/msa/vc-executor-docker/docker/Dockerfile +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -18,6 +18,9 @@ FROM thingsboard/openjdk11 COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ +RUN mkdir -p /home/thingsboard/.config/jgit +RUN chown -R ${pkg.user}:${pkg.user} /home/thingsboard + RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-vc-executor.sh /usr/bin From 5bb9bdface4f9b03c2c9690c6c7e478b32bc05b5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 13:43:35 +0300 Subject: [PATCH 085/178] Cache for VersionControlSettings --- .../service/session/SessionRedisCache.java | 2 +- .../DefaultEntitiesVersionControlService.java | 66 ++----------- ...efaultTbVersionControlSettingsService.java | 98 +++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 2 - .../vc/TbVersionControlSettingsService.java | 31 ++++++ .../VersionControlSettingsCaffeineCache.java | 36 +++++++ .../vc/VersionControlSettingsRedisCache.java | 41 ++++++++ .../src/main/resources/thingsboard.yml | 3 + .../server/common/data/CacheConstants.java | 1 + 9 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java index cd3e8cb56b..dc12326bdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java @@ -33,7 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; public class SessionRedisCache extends RedisTbTransactionalCache { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7ad3e309e0..7f791163a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,9 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -36,7 +34,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -46,7 +43,6 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -61,7 +57,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -86,10 +81,10 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final TbVersionControlSettingsService vcSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AdminSettingsService adminSettingsService; private final TransactionTemplate transactionTemplate; private ListeningExecutorService executor; @@ -311,54 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; + return vcSettingsService.get(tenantId); } @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - EntitiesVersionControlSettings storedSettings = null; - if (adminSettings != null) { - try { - storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } + versionControlSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); try { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.initRepository(tenantId, versionControlSettings).get(); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - return savedVersionControlSettings; + return vcSettingsService.save(tenantId, versionControlSettings); } @Override public void deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + if (vcSettingsService.delete(tenantId)) { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.clearRepository(tenantId).get(); } @@ -366,8 +331,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); - settings = this.restoreCredentials(settings, storedSettings); + settings = this.vcSettingsService.restore(tenantId, settings); try { //TODO: ashvayka: replace future.get with deferred result. gitServiceQueue.testRepository(tenantId, settings).get(); @@ -379,7 +343,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private String getCauseMessage(Exception e) { String message; - if(e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())){ + if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { message = e.getCause().getMessage(); } else { message = e.getMessage(); @@ -387,20 +351,4 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return message; } - private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - if (storedSettings != null) { - settings.setPassword(storedSettings.getPassword()); - } - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - if (storedSettings != null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java new file mode 100644 index 0000000000..a62b046ee2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + + @Override + public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { + EntitiesVersionControlSettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public EntitiesVersionControlSettings get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + }, true); + } + + @Override + public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedVersionControlSettings; + } + + @Override + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 08b92db96a..1d8cfca00c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -36,8 +36,6 @@ import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { - String SETTINGS_KEY = "entitiesVersionControl"; - ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java new file mode 100644 index 0000000000..178499c6c7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; + +public interface TbVersionControlSettingsService { + + EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + EntitiesVersionControlSettings get(TenantId tenantId); + + EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java new file mode 100644 index 0000000000..8cab03ca29 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("VersionControlCache") +public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java new file mode 100644 index 0000000000..9c778d6e4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("VersionControlCache") +public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { + + public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index aaec0852ca..62c8052254 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,6 +433,9 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + vcSettings: + timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 571af34086..85ea116f0a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,4 +31,5 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; + public static final String VC_SETTINGS_CACHE = "vcSettings"; } From ed44304eac4f779251912faf6a34349003f691f4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 16:26:39 +0300 Subject: [PATCH 086/178] UI: Branch select improvements. Entity versions table improvements. --- .../DefaultGitVersionControlQueueService.java | 9 ++ common/cluster-api/src/main/proto/queue.proto | 2 + .../DefaultClusterVersionControlService.java | 11 +- .../server/service/sync/vc/GitRepository.java | 2 +- ui-ngx/src/app/core/http/admin.service.ts | 17 ++- .../http/entities-version-control.service.ts | 45 +++++- .../vc/entity-versions-table.component.html | 24 ++- .../vc/entity-versions-table.component.ts | 33 ++++- .../vc/branch-autocomplete.component.html | 8 +- .../vc/branch-autocomplete.component.ts | 140 ++++++++++-------- .../assets/locale/locale.constant-en_US.json | 3 +- 11 files changed, 210 insertions(+), 84 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..66e0442297 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -150,6 +150,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -159,6 +162,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch).setEntityType(entityType.name()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -171,6 +177,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..82070a5895 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -724,6 +724,8 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; int32 pageSize = 5; int32 page = 6; + string sortProperty = 7; + string sortDirection = 8; } message EntityVersionProto { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..a27a4862bf 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -290,7 +291,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } else { path = null; } - var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage())); + SortOrder sortOrder = null; + if (StringUtils.isNotEmpty(request.getSortProperty())) { + var direction = SortOrder.Direction.DESC; + if (StringUtils.isNotEmpty(request.getSortDirection())) { + direction = SortOrder.Direction.valueOf(request.getSortDirection()); + } + sortOrder = new SortOrder(request.getSortProperty(), direction); + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage(), null, sortOrder)); reply(ctx, Optional.empty(), builder -> builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() .setTotalPages(data.getTotalPages()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..a1a46d5b25 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -281,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 79bb6580b8..7c12bd3a5d 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -26,6 +26,8 @@ import { TestSmsRequest, UpdateMessage } from '@shared/models/settings.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' @@ -33,7 +35,8 @@ import { export class AdminService { constructor( - private http: HttpClient + private http: HttpClient, + private entitiesVersionControlService: EntitiesVersionControlService ) { } public getAdminSettings(key: string, config?: RequestConfig): Observable> { @@ -72,11 +75,19 @@ export class AdminService { public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, config?: RequestConfig): Observable { return this.http.post('/api/admin/vcSettings', versionControlSettings, - defaultHttpOptionsFromConfig(config)); + defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index f8668f2ecf..eba229de67 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,30 +17,63 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { Observable } from 'rxjs'; +import { combineLatest, Observable, of } from 'rxjs'; import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; -import { DeviceInfo } from '@shared/models/device.models'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { createSelector, select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, combineAll, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' }) export class EntitiesVersionControlService { + branchList: Array = null; + constructor( - private http: HttpClient + private http: HttpClient, + private store: Store ) { + + this.store.pipe(select(selectIsUserLoaded)).subscribe( + () => { + this.branchList = null; + } + ); + } + + public clearBranchList(): void { + this.branchList = null; } - public listBranches(config?: RequestConfig): Observable> { - return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + public listBranches(): Observable> { + if (!this.branchList) { + return this.http.get>('/api/entities/vc/branches', + defaultHttpOptionsFromConfig({ignoreErrors: true, ignoreLoading: false})).pipe( + catchError(() => of([] as Array)), + tap((list) => { + this.branchList = list; + }) + ); + } else { + return of(this.branchList); + } } public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { - return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + const branch = request.branch; + if (this.branchList && !this.branchList.find(b => b.name === branch)) { + this.branchList = null; + } + }) + ); } public listEntityVersions(pageLink: PageLink, branch: string, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b47c2fcc79..f5aa4a04d2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -48,27 +48,41 @@ - {{ 'version-control.version-id' | translate }} + {{ 'version-control.version-id' | translate }} - {{ entityVersion.id }} + + + - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} - + - {{ singleEntityMode ? 'version-control.no-entity-versions-text' : 'version-control.no-versions-text' }} + {{ 'common.loading' | translate }}
7) { + versionId = versionId.slice(0, 7); + } + return versionId; + } + private initFromDefaultBranch() { - this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } else { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(true); + } } private updateData() { @@ -164,7 +182,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { - this.branch = null; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; @@ -185,6 +202,8 @@ class EntityVersionsDatasource implements DataSource { public pageData$ = this.pageDataSubject.asObservable(); + public dataLoading = true; + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} connect(collectionViewer: CollectionViewer): Observable> { @@ -199,6 +218,7 @@ class EntityVersionsDatasource implements DataSource { loadEntityVersions(singleEntityMode: boolean, branch: string, externalEntityId: EntityId, pageLink: PageLink): Observable> { + this.dataLoading = true; const result = new ReplaySubject>(); this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( catchError(() => of(emptyPageData())), @@ -207,6 +227,7 @@ class EntityVersionsDatasource implements DataSource { this.entityVersionsSubject.next(pageData.data); this.pageDataSubject.next(pageData); result.next(pageData); + this.dataLoading = false; } ); return result; diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index b189d70199..74672b1772 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -17,12 +17,14 @@ --> {{ 'version-control.branch' | translate }} - + [matAutocomplete]="branchAutocomplete"> - + +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts new file mode 100644 index 0000000000..368a4bf441 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -0,0 +1,84 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-entity-version-export', + templateUrl: './entity-version-export.component.html', + styleUrls: [] +}) +export class EntityVersionExportComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + entityId: EntityId; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + exportFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.exportFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null, null); + } + } + + export(): void { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result, request.branch); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index f5aa4a04d2..11d8b2a2e2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -31,10 +31,12 @@ - diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index 9017de60b8..ec5cb02256 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -78,7 +78,7 @@ tb-branch-autocomplete { mat-form-field { font-size: 16px; - width: 200px; + width: 250px; .mat-form-field-wrapper { padding-bottom: 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 884bab417b..436267fe2f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -21,8 +21,8 @@ import { ElementRef, Input, OnDestroy, - OnInit, - ViewChild + OnInit, Renderer2, + ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -33,7 +33,7 @@ import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, map, tap } from 'rxjs/operators'; -import { EntityVersion } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -42,6 +42,10 @@ import { hidePageSizePixelValue } from '@shared/models/constants'; import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; import { isNotEmptyStr } from '@core/utils'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -68,6 +72,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; + vcExportPopover: TbPopoverComponent; + private componentResize$: ResizeObserver; @Input() @@ -94,12 +100,18 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + @Input() + entityId: EntityId; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private popoverService: TbPopoverService, + private renderer: Renderer2, private cd: ChangeDetectorRef, + private viewContainerRef: ViewContainerRef, private elementRef: ElementRef) { super(store); this.dirtyValue = !this.activeValue; @@ -148,10 +160,36 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } - vcExport($event: Event) { + toggleVcExport($event: Event, exportButton: MatButton) { if ($event) { $event.stopPropagation(); } + const trigger = exportButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + { + branch: this.branch, + entityId: this.entityId, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + this.vcExportPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { + if (!visible) { + this.vcExportPopover = null; + } + }); + } } versionIdContent(entityVersion: EntityVersion): string { diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html deleted file mode 100644 index e7ea79cd73..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html +++ /dev/null @@ -1,77 +0,0 @@ - -
- -

{{ (createResult ? 'version-control.entity-version-exported' : 'version-control.export-entity-version') | translate }}

- - - - - -
-
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
-
-
-
-
- - -
-
- -
-
diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts deleted file mode 100644 index 9d3affea42..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts +++ /dev/null @@ -1,105 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; -import { ErrorStateMatcher } from '@angular/material/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; -import { Router } from '@angular/router'; -import { DialogComponent } from '@app/shared/components/dialog.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - SingleEntityVersionCreateRequest, - VersionCreateRequestType, - VersionCreationResult -} from '@shared/models/vc.models'; -import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; -import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; -import { TranslateService } from '@ngx-translate/core'; - -export interface VcEntityExportDialogData { - entityId: EntityId; -} - -@Component({ - selector: 'tb-vc-entity-export-dialog', - templateUrl: './vc-entity-export-dialog.component.html', - providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], - styleUrls: [] -}) -export class VcEntityExportDialogComponent extends DialogComponent - implements OnInit, ErrorStateMatcher { - - exportFormGroup: FormGroup; - - submitted = false; - - createResult: VersionCreationResult; - - createResultMessage: SafeHtml; - - constructor(protected store: Store, - protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, - @SkipSelf() private errorStateMatcher: ErrorStateMatcher, - public dialogRef: MatDialogRef, - private entitiesVersionControlService: EntitiesVersionControlService, - private translate: TranslateService, - private domSanitizer: DomSanitizer, - private fb: FormBuilder) { - super(store, router, dialogRef); - - this.exportFormGroup = this.fb.group({ - branch: [null, [Validators.required]], - versionName: [null, [Validators.required]], - saveRelations: [false, []] - }); - } - - ngOnInit(): void { - } - - isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { - const originalErrorState = this.errorStateMatcher.isErrorState(control, form); - const customErrorState = !!(control && control.invalid && this.submitted); - return originalErrorState || customErrorState; - } - - cancel(): void { - this.dialogRef.close(); - } - - export(): void { - this.submitted = true; - const request: SingleEntityVersionCreateRequest = { - entityId: this.data.entityId, - branch: this.exportFormGroup.get('branch').value, - versionName: this.exportFormGroup.get('versionName').value, - config: { - saveRelations: this.exportFormGroup.get('saveRelations').value - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - this.createResult = result; - const message = this.translate.instant('version-control.export-entity-version-result-message', - {name: result.version.name, commitId: result.version.id}); - this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); - }); - } -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 903460054b..c631819128 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -21,5 +21,6 @@ diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 07bdbcba25..2ca68255ae 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -44,6 +44,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() externalEntityId: EntityId; + @Input() + entityId: EntityId; + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 1730c69084..ff08af8039 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,11 +22,6 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - VcEntityExportDialogComponent, - VcEntityExportDialogData -} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -46,17 +41,6 @@ export class HomeDialogsService { } } - public exportVcEntity(entityId: EntityId): Observable { - return this.dialog.open(VcEntityExportDialogComponent, - { - disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - entityId - } - }).afterClosed(); - } - private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index e5accc15a9..3a3f2dda92 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 75171b4532..00ed882cd9 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,12 +46,6 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} - - - - - + + + + +
+ + +   + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 436267fe2f..b60665f2b4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -29,10 +29,10 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityId } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; -import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; -import { catchError, map, tap } from 'rxjs/operators'; +import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; @@ -61,6 +61,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni displayedColumns = ['timestamp', 'id', 'name']; pageLink: PageLink; + textSearchMode = false; dataSource: EntityVersionsDatasource; hidePageSize = false; @@ -103,6 +104,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @ViewChild('searchInput') searchInputField: ElementRef; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @@ -148,6 +151,17 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } ngAfterViewInit() { + fromEvent(this.searchInputField.nativeElement, 'keyup') + .pipe( + debounceTime(400), + distinctUntilChanged(), + tap(() => { + this.paginator.pageIndex = 0; + this.updateData(); + }) + ) + .subscribe(); + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); merge(this.sort.sortChange, this.paginator.page) .pipe( @@ -155,7 +169,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni ) .subscribe(); this.viewsInited = true; - if (!this.singleEntityMode) { + if (!this.singleEntityMode || (this.activeValue && this.externalEntityIdValue)) { this.initFromDefaultBranch(); } } @@ -200,6 +214,22 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni return versionId; } + enterFilterMode() { + this.textSearchMode = true; + this.pageLink.textSearch = ''; + setTimeout(() => { + this.searchInputField.nativeElement.focus(); + this.searchInputField.nativeElement.setSelectionRange(0, 0); + }, 10); + } + + exitFilterMode() { + this.textSearchMode = false; + this.pageLink.textSearch = null; + this.paginator.pageIndex = 0; + this.updateData(); + } + private initFromDefaultBranch() { if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { this.paginator.pageIndex = 0; @@ -220,6 +250,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { + this.textSearchMode = false; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index 2970987be7..3462fbb728 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -33,7 +33,7 @@ import { DialogService } from '@core/services/dialog.service'; import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap } from 'rxjs/operators'; +import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; @Component({ @@ -87,6 +87,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On }); this.store.pipe( select(selectHasVersionControl), + take(1), mergeMap((hasVersionControl) => { if (hasVersionControl) { return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( From 905b60da6a4a6d006cfc86b30861358cb2efd190 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 12:32:51 +0300 Subject: [PATCH 091/178] Fix GitRepository.status command, Increased timeout between poll commands. --- application/src/main/resources/thingsboard.yml | 3 +++ .../thingsboard/server/service/sync/vc/GitRepository.java | 5 ++++- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 62c8052254..31e5d7a76e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -931,6 +931,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index a0c905852b..c22e18c004 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -241,7 +241,10 @@ public class GitRepository { public Status status() throws GitAPIException { org.eclipse.jgit.api.Status status = execute(git.status()); - return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + Set modified = new HashSet<>(); + modified.addAll(status.getModified()); + modified.addAll(status.getChanged()); + return new Status(status.getAdded(), modified, status.getRemoved()); } public Commit commit(String message) throws GitAPIException { diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 8c34a91e80..4e430e5ba6 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,6 +73,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" From 46e0c7c90897db26fa30c604bfd7dc6b4aa5a2ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 13:50:50 +0300 Subject: [PATCH 092/178] UI: Implement entity version restore --- .../DefaultEntitiesVersionControlService.java | 2 +- .../http/entities-version-control.service.ts | 12 ++- .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-export.component.html | 95 +++++++++++-------- .../vc/entity-version-export.component.scss | 21 ++++ .../vc/entity-version-export.component.ts | 16 +++- .../vc/entity-version-restore.component.html | 49 ++++++++++ .../vc/entity-version-restore.component.ts | 86 +++++++++++++++++ .../vc/entity-versions-table.component.html | 15 +++ .../vc/entity-versions-table.component.ts | 56 ++++++++--- .../vc/version-control.component.html | 3 +- .../vc/version-control.component.ts | 5 +- .../pages/asset/asset-tabs.component.html | 1 + .../customer/customer-tabs.component.html | 1 + .../dashboard/dashboard-tabs.component.html | 1 + .../device-profile-tabs.component.html | 3 +- .../pages/device/device-tabs.component.html | 1 + .../rulechain/rulechain-tabs.component.html | 1 + ui-ngx/src/app/shared/models/vc.models.ts | 29 ++++++ .../assets/locale/locale.constant-en_US.json | 9 +- 20 files changed, 347 insertions(+), 66 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 825171493e..e812595ca4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -190,7 +190,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - Futures.transform(future, entityData -> { + return Futures.transform(future, entityData -> { EntityImportResult importResult = transactionTemplate.execute(status -> { try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index eba229de67..86674f0569 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,13 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { combineLatest, Observable, of } from 'rxjs'; -import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { + BranchInfo, + EntityVersion, + VersionCreateRequest, + VersionCreationResult, + VersionLoadRequest, VersionLoadResult +} from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -95,4 +101,8 @@ export class EntitiesVersionControlService { return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public loadEntitiesVersion(request: VersionLoadRequest, config?: RequestConfig): Observable> { + return this.http.post>('/api/entities/vc/entity', request, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 38cf85e72c..6134708176 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -157,6 +157,7 @@ import { VersionControlSettingsComponent } from '@home/components/vc/version-con import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @NgModule({ declarations: @@ -284,7 +285,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], imports: [ CommonModule, @@ -406,7 +408,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html index 5791790b37..3838ea111d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html @@ -16,45 +16,58 @@ -->
- -

{{ 'version-control.create-entity-version' | translate }}

- - - - -
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
- - -
+
+ +

{{ 'version-control.create-entity-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+ + +
+
+
+
{{ resultMessage }}
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss new file mode 100644 index 0000000000..46e55cf4d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .export-result-message { + padding: 48px 8px 8px; + text-align: center; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts index 368a4bf441..2aa3123858 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -26,11 +26,12 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; @Component({ selector: 'tb-entity-version-export', templateUrl: './entity-version-export.component.html', - styleUrls: [] + styleUrls: ['./entity-version-export.component.scss'] }) export class EntityVersionExportComponent extends PageComponent implements OnInit { @@ -43,10 +44,16 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; + @Input() + onContentUpdated: () => void; + exportFormGroup: FormGroup; + resultMessage: string; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, private fb: FormBuilder) { super(store); } @@ -76,7 +83,12 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni type: VersionCreateRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { this.onClose(result, request.branch); } }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html new file mode 100644 index 0000000000..cb71696b4e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -0,0 +1,49 @@ + +
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + +
+
+
+ + {{ 'version-control.load-entity-relations' | translate }} + +
+
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts new file mode 100644 index 0000000000..2cc615c011 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -0,0 +1,86 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { SingleEntityVersionLoadRequest, VersionLoadRequestType, VersionLoadResult } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-entity-version-restore', + templateUrl: './entity-version-restore.component.html', + styleUrls: [] +}) +export class EntityVersionRestoreComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: (result: Array | null) => void; + + restoreFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.restoreFormGroup = this.fb.group({ + loadRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + restore(): void { + const request: SingleEntityVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + config: { + loadRelations: this.restoreFormGroup.get('loadRelations').value + }, + type: VersionLoadRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 5acfe3b0ba..2e5302881e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -99,6 +99,21 @@ {{ entityVersion.name }} + + + + +
+ +
+ + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index b60665f2b4..06c778d278 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -18,10 +18,10 @@ import { AfterViewInit, ChangeDetectorRef, Component, - ElementRef, + ElementRef, EventEmitter, Input, OnDestroy, - OnInit, Renderer2, + OnInit, Output, Renderer2, ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; @@ -33,7 +33,7 @@ import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; -import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult, VersionLoadResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -46,6 +46,7 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { MatButton } from '@angular/material/button'; import { TbPopoverComponent } from '@shared/components/popover.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @Component({ selector: 'tb-entity-versions-table', @@ -59,7 +60,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name']; + displayedColumns = ['timestamp', 'id', 'name', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; @@ -73,8 +74,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; - vcExportPopover: TbPopoverComponent; - private componentResize$: ResizeObserver; @Input() @@ -104,6 +103,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + @ViewChild('searchInput') searchInputField: ElementRef; @ViewChild(MatPaginator) paginator: MatPaginator; @@ -182,13 +184,13 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { - this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + const vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'left', true, null, { branch: this.branch, entityId: this.entityId, onClose: (result: VersionCreationResult | null, branch: string | null) => { - this.vcExportPopover.hide(); + vcExportPopover.hide(); if (result) { if (this.branch !== branch) { this.branchChanged(branch); @@ -196,13 +198,39 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } + }, + onContentUpdated: () => { + vcExportPopover.updatePosition(); + setTimeout(() => { + vcExportPopover.updatePosition(); + }); + } + }, {}, {}, {}, false); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + externalEntityId: this.externalEntityIdValue, + onClose: (result: Array | null) => { + restoreVersionPopover.hide(); + if (result && result.length) { + this.versionRestored.emit(); + } } }, {}, {}, {}, false); - this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { - if (!visible) { - this.vcExportPopover = null; - } - }); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index c631819128..e5e650ac85 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -22,5 +22,6 @@ + [externalEntityId]="externalEntityId" + (versionRestored)="versionRestored.emit()"> diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 2ca68255ae..da6828c4a0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; @@ -47,6 +47,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 3a3f2dda92..c115a43ab6 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html index 7524fd59f7..634443a397 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html index 21b20bcb82..61a030ac16 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html @@ -22,5 +22,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index e713e70690..effac9a7c8 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -75,8 +75,9 @@ label="{{ 'audit-log.audit-logs' | translate }}"> - diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index d84306d029..78d494d8ae 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html index c60910b565..49a2d83372 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html @@ -55,5 +55,6 @@ diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index d574a1b55f..4975b8ea2b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -15,16 +15,26 @@ /// import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; export interface VersionCreateConfig { saveRelations: boolean; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionCreateRequest { versionName: string; branch: string; @@ -37,6 +47,18 @@ export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { type: VersionCreateRequestType.SINGLE_ENTITY; } +export interface VersionLoadRequest { + branch: string; + versionId: string; + type: VersionLoadRequestType; +} + +export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { + externalEntityId: EntityId; + config: VersionLoadConfig; + type: VersionLoadRequestType.SINGLE_ENTITY; +} + export interface BranchInfo { name: string; default: boolean; @@ -54,3 +76,10 @@ export interface VersionCreationResult { modified: number; removed: number; } + +export interface VersionLoadResult { + entityType: EntityType; + created: number; + updated: number; + deleted: number; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 202597ade1..0a2f39bcd4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -57,7 +57,8 @@ "download": "Download", "next-with-label": "Next: {{label}}", "read-more": "Read more", - "hide": "Hide" + "hide": "Hide", + "restore": "Restore" }, "aggregation": { "aggregation": "Aggregation", @@ -3126,7 +3127,11 @@ "no-entity-versions-text": "No entity versions found", "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", - "create-version": "Create version" + "create-version": "Create version", + "nothing-to-commit": "Nothing to commit", + "restore-version": "Restore version", + "restore-entity-from-version": "Restore entity from version '{{versionName}}'", + "load-entity-relations": "Load entity relations" }, "widget": { "widget-library": "Widgets Library", From 6558b0b24d2d4a4baae5edaefab987cd1f38fe96 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:27:22 +0300 Subject: [PATCH 093/178] UI: Improve restore action --- .../home/components/vc/entity-versions-table.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 06c778d278..fcaa12f352 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -27,7 +27,7 @@ import { import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { EntityId } from '@shared/models/id/entity-id'; +import { EntityId, entityIdEquals } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; @@ -91,7 +91,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() set externalEntityId(externalEntityId: EntityId) { - if (this.externalEntityIdValue !== externalEntityId) { + if (!entityIdEquals(this.externalEntityIdValue, externalEntityId)) { this.externalEntityIdValue = externalEntityId; this.resetSortAndFilter(this.activeValue); if (!this.activeValue) { From c46b5b6448e81bca42bd4a7afde261ad6f030ac4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:34:25 +0300 Subject: [PATCH 094/178] Fix message --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0a2f39bcd4..24cdff2735 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3128,7 +3128,7 @@ "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", "create-version": "Create version", - "nothing-to-commit": "Nothing to commit", + "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations" From e4ed8c508098849efdbef94834d7e31e3a3235e2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 26 May 2022 15:10:46 +0300 Subject: [PATCH 095/178] Api for comparing current entity data to specific version; refactoring --- .../EntitiesVersionControlController.java | 13 +++ .../DefaultEntitiesExportImportService.java | 10 +- .../DefaultEntitiesVersionControlService.java | 32 +++++- .../DefaultGitVersionControlQueueService.java | 83 ++++++++++---- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 6 + .../{ => data}/ClearRepositoryGitRequest.java | 2 +- .../sync/vc/{ => data}/CommitGitRequest.java | 2 +- .../sync/vc/data/ContentsDiffGitRequest.java | 33 ++++++ .../{ => data}/EntitiesContentGitRequest.java | 3 +- .../{ => data}/EntityContentGitRequest.java | 2 +- .../vc/{ => data}/ListBranchesGitRequest.java | 3 +- .../vc/{ => data}/ListEntitiesGitRequest.java | 3 +- .../vc/{ => data}/ListVersionsGitRequest.java | 6 +- .../sync/vc/{ => data}/PendingGitRequest.java | 2 +- .../sync/vc/data/VersionsDiffGitRequest.java | 38 ++++++ .../sync/vc/{ => data}/VoidGitRequest.java | 5 +- common/cluster-api/src/main/proto/queue.proto | 32 ++++++ .../common/data/sync/vc/EntityDataDiff.java | 28 +++++ .../data/sync/vc/EntityVersionsDiff.java | 34 ++++++ .../load/EntityTypeVersionLoadConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 - .../thingsboard/common/util/JacksonUtil.java | 13 ++- .../DefaultClusterVersionControlService.java | 39 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 21 ++-- .../server/service/sync/vc/GitRepository.java | 108 ++++++++++++------ .../service/sync/vc/GitRepositoryService.java | 5 + 27 files changed, 433 insertions(+), 97 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ClearRepositoryGitRequest.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/CommitGitRequest.java (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntitiesContentGitRequest.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntityContentGitRequest.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListBranchesGitRequest.java (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListEntitiesGitRequest.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListVersionsGitRequest.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/PendingGitRequest.java (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/VoidGitRequest.java (85%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bafc93c19f..793ab0503c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -225,6 +226,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") + public DeferredResult compareEntityDataToVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid, + @RequestParam String versionId) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } @ApiOperation(value = "", notes = "" + "SINGLE_ENTITY:" + NEW_LINE + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 38d3b00213..777e84f348 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -24,18 +24,18 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7f791163a0..73eaafe842 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -57,11 +60,13 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -75,6 +80,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.transform; +import static com.google.common.util.concurrent.Futures.transformAsync; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -109,7 +117,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - return Futures.transformAsync(pendingCommit, commit -> { + return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { @@ -146,7 +154,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont break; } } - return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + return transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); }, executor); } @@ -195,7 +203,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) + .findExistingByName(false) .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); @@ -298,6 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } + @Override + public ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception { + HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); + + EntityId externalId = ((ExportableEntity) entity).getExternalId(); + if (externalId == null) externalId = entityId; + + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(true) + .build()); + return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion), + JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { + return new EntityDataDiff(currentVersion, otherVersion, rawDiff); + }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + } @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..270dc5b38c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; @@ -24,6 +22,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -36,6 +35,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -54,12 +54,23 @@ import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest; +import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; +import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -74,7 +85,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final DefaultEntitiesVersionControlService entitiesVersionControlService; private final Map> pendingRequestMap = new HashMap<>(); - private final ObjectMapper jsonMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, @@ -101,13 +111,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson; - try { - entityDataJson = jsonMapper.writeValueAsString(entityData); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } + String entityDataJson = JacksonUtil.toPrettyString(entityData); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -176,10 +180,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg)); } @Override @@ -201,19 +202,32 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg)); } @Override public ListenableFuture> listBranches(TenantId tenantId) { ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); + } - registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + @Override + public ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) { + String path = entityType != null ? getRelativePath(entityType, externalId) : ""; + VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2); + return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder() + .setPath(request.getPath()) + .setVersionId1(request.getVersionId1()) + .setVersionId2(request.getVersionId2()) + .build())); + } - return request.getFuture(); + @Override + public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { + ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); + return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() + .setContent1(content1) + .setContent2(content2))); } @Override @@ -246,6 +260,14 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + private ListenableFuture sendRequest(PendingGitRequest request, Consumer enrichFunction) { + registerAndSend(request, builder -> { + enrichFunction.accept(builder); + return builder.build(); + }, wrap(request.getFuture())); + return request.getFuture(); + } + @Override @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { @@ -334,6 +356,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); ((EntitiesContentGitRequest) request).getFuture() .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + } else if (vcResponseMsg.hasVersionsDiffResponse()) { + TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); + List entityVersionsDiffList = diffResponse.getDiffList().stream() + .map(diff -> EntityVersionsDiff.builder() + .externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()), + new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB()))) + .entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ? + toData(diff.getEntityDataAtVersion1()) : null) + .entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ? + toData(diff.getEntityDataAtVersion2()) : null) + .rawDiff(diff.getRawDiff()) + .build()) + .collect(Collectors.toList()); + ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); + } else if (vcResponseMsg.hasContentsDiffResponse()) { + String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); + ((ContentsDiffGitRequest) request).getFuture().set(diff); } } } @@ -354,7 +393,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @SuppressWarnings("rawtypes") @SneakyThrows private EntityExportData toData(String data) { - return jsonMapper.readValue(data, EntityExportData.class); + return JacksonUtil.fromString(data, EntityExportData.class); } private static TbQueueCallback wrap(SettableFuture future) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 1d8cfca00c..cb5ec52b6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; -import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -50,6 +49,8 @@ public interface EntitiesVersionControlService { ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index 00899a94e2..b7eb368055 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import java.util.List; @@ -58,6 +60,10 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); + + ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java index 86472e7882..09245eb895 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java index 3c83a34cfe..a989439d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java new file mode 100644 index 0000000000..7c3fb7a600 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class ContentsDiffGitRequest extends PendingGitRequest { + + private final String content1; + private final String content2; + + public ContentsDiffGitRequest(TenantId tenantId, String content1, String content2) { + super(tenantId); + this.content1 = content1; + this.content2 = content2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java index 2ded932e44..ad653edd0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java index c9ce3c76af..6a1d60a065 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java index c8219641da..c045030dc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java index 0c7216ef49..3fc531735e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java index 0c646b424c..d40a589646 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java @@ -13,15 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; - -import java.util.List; public class ListVersionsGitRequest extends PendingGitRequest> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java index e63ba04f45..fbec600ed0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java new file mode 100644 index 0000000000..e73706609f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +@Getter +public class VersionsDiffGitRequest extends PendingGitRequest> { + + private final String path; + private final String versionId1; + private final String versionId2; + + public VersionsDiffGitRequest(TenantId tenantId, String path, String versionId1, String versionId2) { + super(tenantId); + this.path = path; + this.versionId1 = versionId1; + this.versionId2 = versionId2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java index d829960ab4..c48bfa7a03 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java @@ -13,12 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; - -import java.util.List; public class VoidGitRequest extends PendingGitRequest { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..5a61dfa46a 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -784,6 +784,34 @@ message EntitiesContentResponseMsg { repeated string data = 1; } +message VersionsDiffRequestMsg { + string path = 1; + string versionId1 = 2; + string versionId2 = 3; +} + +message VersionsDiffResponseMsg { + repeated EntityVersionsDiff diff = 1; +} + +message EntityVersionsDiff { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + string entityDataAtVersion1 = 4; + string entityDataAtVersion2 = 5; + string rawDiff = 6; +} + +message ContentsDiffRequestMsg { + string content1 = 1; + string content2 = 2; +} + +message ContentsDiffResponseMsg { + string diff = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -804,6 +832,8 @@ message ToVersionControlServiceMsg { ListBranchesRequestMsg listBranchesRequest = 13; EntityContentRequestMsg entityContentRequest = 14; EntitiesContentRequestMsg entitiesContentRequest = 15; + VersionsDiffRequestMsg versionsDiffRequest = 16; + ContentsDiffRequestMsg contentsDiffRequest = 17; } message VersionControlResponseMsg { @@ -817,6 +847,8 @@ message VersionControlResponseMsg { ListVersionsResponseMsg listVersionsResponse = 8; EntityContentResponseMsg entityContentResponse = 9; EntitiesContentResponseMsg entitiesContentResponse = 10; + VersionsDiffResponseMsg versionsDiffResponse = 11; + ContentsDiffResponseMsg contentsDiffResponse = 12; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java new file mode 100644 index 0000000000..800583b0b8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +public class EntityDataDiff { + private EntityExportData currentVersion; + private EntityExportData otherVersion; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java new file mode 100644 index 0000000000..2104472047 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionsDiff { + private EntityId externalId; + private EntityExportData entityDataAtVersion1; + private EntityExportData entityDataAtVersion2; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 837986f82d..ffb1f25d3c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -23,5 +23,6 @@ import lombok.EqualsAndHashCode; public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; + private boolean findExistingEntityByName; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 31b34dbcb7..8ed9c34694 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,6 +21,5 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; - private boolean findExistingEntityByName; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..b6e2b54479 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -19,12 +19,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -32,6 +31,8 @@ import java.util.Set; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -96,6 +97,14 @@ public class JacksonUtil { } } + public static String toPrettyString(Object o) { + try { + return PRETTY_JSON_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public static JsonNode toJsonNode(String value) { if (value == null || value.isEmpty()) { return null; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 5fcb9985bd..47ab9fc2de 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -31,6 +31,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -91,6 +92,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.service.sync.vc.DefaultGitRepositoryService.fromRelativePath; + @Slf4j @TbVersionControlComponent @Service @@ -237,17 +240,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe vcService.fetch(ctx.getTenantId()); handleListBranches(ctx, msg.getListBranchesRequest()); } else if (msg.hasListEntitiesRequest()) { - vcService.fetch(ctx.getTenantId()); handleListEntities(ctx, msg.getListEntitiesRequest()); } else if (msg.hasListVersionRequest()) { vcService.fetch(ctx.getTenantId()); handleListVersions(ctx, msg.getListVersionRequest()); } else if (msg.hasEntityContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntityContentRequest(ctx, msg.getEntityContentRequest()); } else if (msg.hasEntitiesContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } else if (msg.hasVersionsDiffRequest()) { + handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); + } else if (msg.hasContentsDiffRequest()) { + handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); } } } @@ -322,6 +326,31 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); } + private void handleVersionsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.VersionsDiffRequestMsg request) throws IOException { + List diffList = vcService.getVersionsDiffList(ctx.getTenantId(), request.getPath(), request.getVersionId1(), request.getVersionId2()).stream() + .map(diff -> { + EntityId entityId = fromRelativePath(diff.getFilePath()); + return TransportProtos.EntityVersionsDiff.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityDataAtVersion1(diff.getFileContentAtCommit1()) + .setEntityDataAtVersion2(diff.getFileContentAtCommit2()) + .setRawDiff(diff.getDiffStringValue()) + .build(); + }) + .collect(Collectors.toList()); + + reply(ctx, builder -> builder.setVersionsDiffResponse(TransportProtos.VersionsDiffResponseMsg.newBuilder() + .addAllDiff(diffList))); + } + + private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { + String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); + reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() + .setDiff(diff))); + } + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); @@ -430,6 +459,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, e, null); } + private void reply(VersionControlRequestCtx ctx, Function enrichFunction) { + reply(ctx, Optional.empty(), enrichFunction); + } + private void reply(VersionControlRequestCtx ctx, Optional e, Function enrichFunction) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); VersionControlResponseMsg.Builder builder = VersionControlResponseMsg.newBuilder() diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 45c5e563a6..a759ddbfa8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import javax.annotation.PostConstruct; import java.io.File; @@ -167,6 +168,18 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return repository.getFileContentAtCommit(relativePath, versionId); } + @Override + public List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getDiffList(versionId1, versionId2, path); + } + + @Override + public String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getContentsDiff(content1, content2); + } + @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); @@ -178,12 +191,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - private GitRepository checkRepository(TenantId tenantId) { return Optional.ofNullable(repositories.get(tenantId)) .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); @@ -251,7 +258,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } - private EntityId fromRelativePath(String path) { + public static EntityId fromRelativePath(String path) { EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..921630a4ba 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -31,6 +31,12 @@ import org.eclipse.jgit.api.LsRemoteCommand; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.EditList; +import org.eclipse.jgit.diff.HistogramDiff; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -45,6 +51,7 @@ import org.eclipse.jgit.transport.sshd.JGitKeyCache; import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; @@ -54,6 +61,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -207,7 +215,7 @@ public class GitRepository { RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { if (treeWalk == null) { - throw new IllegalArgumentException("Not found"); + throw new IllegalArgumentException("File not found"); } ObjectId blobId = treeWalk.getObjectId(0); try (ObjectReader objectReader = git.getRepository().newObjectReader()) { @@ -248,40 +256,65 @@ public class GitRepository { .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } + public String getContentsDiff(String content1, String content2) throws IOException { + RawText rawContent1 = new RawText(content1.getBytes()); + RawText rawContent2 = new RawText(content2.getBytes()); -// public List getCommitChanges(Commit commit) throws IOException, GitAPIException { -// RevCommit revCommit = resolveCommit(commit.getId()); -// if (revCommit.getParentCount() == 0) { -// return null; // just takes the first parent of a commit, but should find a parent in branch provided -// } -// return execute(git.diff() -// .setOldTree(prepareTreeParser(git.getRepository().parseCommit(revCommit.getParent(0)))) -// .setNewTree(prepareTreeParser(revCommit))).stream() -// .map(diffEntry -> new Diff(diffEntry.getChangeType().name(), diffEntry.getOldPath(), diffEntry.getNewPath())) -// .collect(Collectors.toList()); -// } -// -// -// private AbstractTreeIterator prepareTreeParser(RevCommit revCommit) throws IOException { -// // from the commit we can build the tree which allows us to construct the TreeParser -// //noinspection Duplicates -// org.eclipse.jgit.lib.Repository repository = git.getRepository(); -// try (RevWalk walk = new RevWalk(repository)) { -// RevTree tree = walk.parseTree(revCommit.getTree().getId()); -// -// CanonicalTreeParser treeParser = new CanonicalTreeParser(); -// try (ObjectReader reader = repository.newObjectReader()) { -// treeParser.reset(reader, tree.getId()); -// } -// -// walk.dispose(); -// -// return treeParser; -// } -// } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + + EditList edits = new EditList(); + edits.addAll(new HistogramDiff().diff(RawTextComparator.DEFAULT, rawContent1, rawContent2)); + diffFormatter.format(edits, rawContent1, rawContent2); + return out.toString(); + } + + public List getDiffList(String commit1, String commit2, String path) throws IOException { + ObjectReader reader = git.getRepository().newObjectReader(); + + CanonicalTreeParser tree1Iter = new CanonicalTreeParser(); + ObjectId tree1 = resolveCommit(commit1).getTree(); + tree1Iter.reset(reader, tree1); + + CanonicalTreeParser tree2Iter = new CanonicalTreeParser(); + ObjectId tree2 = resolveCommit(commit2).getTree(); + tree2Iter.reset(reader, tree2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + if (StringUtils.isNotEmpty(path)) { + diffFormatter.setPathFilter(PathFilter.create(path)); + } + + return diffFormatter.scan(tree1, tree2).stream() + .map(diffEntry -> { + Diff diff = new Diff(); + try { + out.reset(); + diffFormatter.format(diffEntry); + diff.setDiffStringValue(out.toString()); + diff.setFilePath(diffEntry.getChangeType() != DiffEntry.ChangeType.DELETE ? diffEntry.getNewPath() : diffEntry.getOldPath()); + diff.setChangeType(diffEntry.getChangeType()); + try { + diff.setFileContentAtCommit1(getFileContentAtCommit(diff.getFilePath(), commit1)); + } catch (IllegalArgumentException ignored) { + } + try { + diff.setFileContentAtCommit2(getFileContentAtCommit(diff.getFilePath(), commit2)); + } catch (IllegalArgumentException ignored) { + } + return diff; + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000L, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -309,7 +342,7 @@ public class GitRepository { return null; }; - private static PageData iterableToPageData (Iterable iterable, + private static PageData iterableToPageData(Iterable iterable, Function mapper, PageLink pageLink, Function> comparatorFunction) { @@ -406,4 +439,13 @@ public class GitRepository { private final Set removed; } + @Data + public static class Diff { + private String filePath; + private DiffEntry.ChangeType changeType; + private String fileContentAtCommit1; + private String fileContentAtCommit2; + private String diffStringValue; + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 9d57eee3ec..ec69e0c80e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import java.io.IOException; import java.util.List; @@ -60,5 +61,9 @@ public interface GitRepositoryService { String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException; + + String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException; + void fetch(TenantId tenantId) throws GitAPIException; } From 576c34b41ae13d150f18dcbe7703c32e3add083f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 15:28:45 +0300 Subject: [PATCH 096/178] Delete repository on Tenant Deletion --- .../server/service/entitiy/tenant/DefaultTbTenantService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 2a8fc1bf0a..22baed8366 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.queue.TbQueueService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.Collections; +import java.util.concurrent.TimeUnit; @Service @TbCoreComponent @@ -38,6 +40,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T private final InstallScripts installScripts; private final TbQueueService tbQueueService; private final TenantProfileService tenantProfileService; + private final EntitiesVersionControlService versionControlService; @Override public Tenant save(Tenant tenant) throws ThingsboardException { @@ -70,6 +73,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T tenantService.deleteTenant(tenantId); tenantProfileCache.evict(tenantId); notificationEntityService.notifyDeleteTenant(tenant); + versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES); } catch (Exception e) { throw handleException(e); } From e86b37b4afe464fcd39475bd82d5d2a9943011ff Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 27 May 2022 13:14:18 +0300 Subject: [PATCH 097/178] UI: Initial implementation of entity versions diff component --- ui-ngx/angular.json | 6 +- ui-ngx/package.json | 2 + .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-diff.component.html | 35 +++++++ .../vc/entity-version-diff.component.scss | 23 +++++ .../vc/entity-version-diff.component.ts | 92 +++++++++++++++++++ .../vc/entity-versions-table.component.html | 9 +- .../vc/entity-versions-table.component.ts | 23 +++++ .../src/app/shared/models/ace/ace.models.ts | 16 ++++ .../assets/locale/locale.constant-en_US.json | 4 +- ui-ngx/yarn.lock | 12 +++ 11 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index f4d4412723..da4d21a84b 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -79,7 +79,8 @@ "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css", "node_modules/@geoman-io/leaflet-geoman-free/dist/leaflet-geoman.css", "node_modules/prismjs/themes/prism.css", - "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css" + "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css", + "node_modules/ace-diff/dist/ace-diff.min.css" ], "stylePreprocessorOptions": { "includePaths": [ @@ -130,7 +131,8 @@ "jstree", "qrcode", "wcwidth", - "leaflet-polylinedecorator" + "leaflet-polylinedecorator", + "ace-diff" ] }, "configurations": { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index e54606dc08..fa21cc9525 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -41,6 +41,7 @@ "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", "ace-builds": "^1.4.13", + "ace-diff": "^3.0.3", "angular-gridster2": "~12.1.1", "angular2-hotkeys": "^2.4.0", "canvas-gauges": "^2.1.7", @@ -105,6 +106,7 @@ "@angular/compiler-cli": "^12.2.13", "@angular/language-service": "^12.2.13", "@ngtools/webpack": "~12.2.13", + "@types/ace-diff": "^2.1.1", "@types/canvas-gauges": "^2.1.4", "@types/flot": "^0.0.32", "@types/jasmine": "~3.10.2", diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 6134708176..9afe26d8f1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -158,6 +158,7 @@ import { VersionControlComponent } from '@home/components/vc/version-control.com import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; +import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; @NgModule({ declarations: @@ -286,7 +287,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], imports: [ CommonModule, @@ -409,7 +411,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html new file mode 100644 index 0000000000..f080fcaf99 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html @@ -0,0 +1,35 @@ + +
+ +

{{ 'version-control.diff-entity-with-version' | translate: {versionName} }}

+ + + + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss new file mode 100644 index 0000000000..5db7fd40e5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .diff-viewer { + position: relative; + border: 1px solid #c0c0c0; + margin-top: 16px; + margin-bottom: 16px; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts new file mode 100644 index 0000000000..f46375b152 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -0,0 +1,92 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { getAceDiff } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-entity-version-diff', + templateUrl: './entity-version-diff.component.html', + styleUrls: ['./entity-version-diff.component.scss'] +}) +export class EntityVersionDiffComponent extends PageComponent implements OnInit, OnDestroy { + + @ViewChild('diffViewer', {static: true}) + diffViewerElmRef: ElementRef; + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: () => void; + + differ: AceDiff; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + getAceDiff().subscribe((aceDiff) => { + this.differ = new aceDiff.default( + { + element: this.diffViewerElmRef.nativeElement, + left: { + copyLinkEnabled: false, + editable: false, + content: 'Left content!' + }, + right: { + copyLinkEnabled: false, + editable: false, + content: 'Right content!' + } + } as AceDiff.AceDiffConstructorOpts + ); + }); + } + + ngOnDestroy(): void { + if (this.differ) { + this.differ.destroy(); + } + } + + close(): void { + if (this.onClose) { + this.onClose(); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 2e5302881e..d931f2f9aa 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -100,10 +100,17 @@ - +
+ + + + {{ 'version-control.differences' | translate : {count: diffCount} }} + + + - - -
-
+
+
{{ 'version-control.current' | translate }}
+
+
{{ versionIdContent() }}
+
+ +
+
+ +
+ +
+
{{ resultMessage }}
+
+ +
+
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss rename to ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss index 46e55cf4d4..e8b3a5fbf6 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .export-result-message { + .create-result-message { padding: 48px 8px 8px; text-align: center; } diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts new file mode 100644 index 0000000000..c152e6b447 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + ComplexVersionCreateRequest, + createDefaultEntityTypesVersionCreate, + SyncStrategy, syncStrategyTranslationMap, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-complex-version-create', + templateUrl: './complex-version-create.component.html', + styleUrls: ['./complex-version-create.component.scss'] +}) +export class ComplexVersionCreateComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + onContentUpdated: () => void; + + createVersionFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + resultMessage: string; + + versionCreateResult: VersionCreationResult = null; + + versionCreateBranch: string = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + syncStrategy: [SyncStrategy.MERGE, Validators.required], + entityTypes: [createDefaultEntityTypesVersionCreate(), []], + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionCreateResult, this.versionCreateBranch); + } + } + + export(): void { + const request: ComplexVersionCreateRequest = { + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + syncStrategy: this.createVersionFormGroup.get('syncStrategy').value, + entityTypes: this.createVersionFormGroup.get('entityTypes').value, + type: VersionCreateRequestType.COMPLEX + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified && !result.removed) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + } else { + this.resultMessage = this.translate.instant('version-control.version-create-result', + {added: result.added, modified: result.modified, removed: result.removed}); + } + this.versionCreateResult = result; + this.versionCreateBranch = request.branch; + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html new file mode 100644 index 0000000000..74cfe9b891 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -0,0 +1,111 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss new file mode 100644 index 0000000000..f90ad2a803 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-create { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts new file mode 100644 index 0000000000..bf2182dfcd --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -0,0 +1,247 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + SyncStrategy, + syncStrategyTranslationMap +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-entity-types-version-create', + templateUrl: './entity-types-version-create.component.html', + styleUrls: ['./entity-types-version-create.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + } + ] +}) +export class EntityTypesVersionCreateComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionCreateConfig}; + + private propagateChange = null; + + public entityTypesVersionCreateFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionCreateFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), [Validators.min(1)]] + }); + this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionCreateFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionCreateFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionCreateFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionCreateFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + syncStrategy: [config.syncStrategy === null ? 'default' : config.syncStrategy, []], + saveRelations: [config.saveRelations, []], + allEntities: [config.allEntities, []], + entityIds: [config.entityIds, [Validators.required]] + }) + } + ); + this.updateEntityTypeValidators(entityTypeControl); + entityTypeControl.get('config').get('allEntities').valueChanges.subscribe(() => { + this.updateEntityTypeValidators(entityTypeControl); + }); + return entityTypeControl; + } + + private updateEntityTypeValidators(entityTypeControl: AbstractControl): void { + const allEntities: boolean = entityTypeControl.get('config').get('allEntities').value; + if (allEntities) { + entityTypeControl.get('config').get('entityIds').disable({emitEvent: false}); + } else { + entityTypeControl.get('config').get('entityIds').enable({emitEvent: false}); + } + entityTypeControl.get('config').get('entityIds').updateValueAndValidity({emitEvent: false}); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionCreateConfig = { + syncStrategy: null, + saveRelations: false, + allEntities: true, + entityIds: [] + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: EntityTypeVersionCreateConfig = entityTypeControl.get('config').value; + let count = config?.entityIds?.length; + if (!isDefinedAndNotNull(count)) { + count = 0; + } + if (entityType) { + return this.translate.instant((config?.allEntities ? entityTypeTranslations.get(entityType).typePlural + : entityTypeTranslations.get(entityType).list), { count }); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionCreateConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + if ((modelValue[val.entityType].syncStrategy as any) === 'default') { + modelValue[val.entityType].syncStrategy = null; + } + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html similarity index 88% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html rename to ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 3838ea111d..5c575b93d4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -24,7 +24,7 @@ -
+
version-control.version-name - + {{ 'version-control.version-name-required' | translate }} @@ -54,13 +54,13 @@
-
{{ resultMessage }}
+
{{ resultMessage }}
+
+ formGroupName="config" style="min-height: 76px;"> {{ 'version-control.all-entities' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index ca0a8cbe85..8276e12d63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -186,7 +186,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const createVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionCreateComponent, 'left', true, null, + this.viewContainerRef, EntityVersionCreateComponent, 'leftTop', true, null, { branch: this.branch, entityId: this.entityId, @@ -251,7 +251,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const diffVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionDiffComponent, 'left', true, null, + this.viewContainerRef, EntityVersionDiffComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, @@ -275,7 +275,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, From 89c04a3460f1417d66594b714deb6f2ca1396c10 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 11:33:49 +0300 Subject: [PATCH 103/178] UI: Implement complex version load. --- .../home/components/home-components.module.ts | 10 +- .../vc/complex-version-load.component.html | 65 ++++++ .../vc/complex-version-load.component.scss | 27 +++ .../vc/complex-version-load.component.ts | 108 +++++++++ .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 90 ++++++++ .../entity-types-version-load.component.scss | 64 ++++++ .../vc/entity-types-version-load.component.ts | 210 ++++++++++++++++++ .../vc/entity-versions-table.component.html | 10 +- .../vc/entity-versions-table.component.ts | 22 ++ ui-ngx/src/app/shared/models/vc.models.ts | 37 ++- .../assets/locale/locale.constant-en_US.json | 10 +- 12 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f0dabc0309..e9fc3f8c9a 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -161,6 +161,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; +import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @NgModule({ declarations: @@ -292,7 +294,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], imports: [ CommonModule, @@ -418,7 +422,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html new file mode 100644 index 0000000000..200dac7f04 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -0,0 +1,65 @@ + +
+
+ +

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

+ + + + + +
+
+ + +
+
+ +
+ + +
+
+
+
+ {{ 'version-control.no-entities-restored' | translate }} +
+
{{ versionLoadResultMessage(versionLoadResult) }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss new file mode 100644 index 0000000000..eaeb773b2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .load-result-message { + padding: 0 8px; + text-align: center; + &:first-child { + padding-top: 48px; + } + &:nth-last-child(2) { + padding-bottom: 8px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts new file mode 100644 index 0000000000..4b785fd384 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -0,0 +1,108 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + createDefaultEntityTypesVersionLoad, + EntityTypeVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-complex-version-load', + templateUrl: './complex-version-load.component.html', + styleUrls: ['./complex-version-load.component.scss'] +}) +export class ComplexVersionLoadComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + onClose: (result: Array | null) => void; + + @Input() + onContentUpdated: () => void; + + loadVersionFormGroup: FormGroup; + + versionLoadResults: Array = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.loadVersionFormGroup = this.fb.group({ + entityTypes: [createDefaultEntityTypesVersionLoad(), []], + }); + } + + versionLoadResultMessage(result: VersionLoadResult): string { + const entityType = result.entityType; + let message = this.translate.instant(entityTypeTranslations.get(entityType).typePlural) + ': '; + const resultMessages: string[] = []; + if (result.created) { + resultMessages.push(this.translate.instant('version-control.created', {created: result.created})); + } + if (result.updated) { + resultMessages.push(this.translate.instant('version-control.updated', {updated: result.updated})); + } + if (result.deleted) { + resultMessages.push(this.translate.instant('version-control.deleted', {deleted: result.deleted})); + } + message += resultMessages.join(', ') + '.'; + return message; + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionLoadResults); + } + } + + restore(): void { + const request: EntityTypeVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + type: VersionLoadRequestType.ENTITY_TYPE + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + this.versionLoadResults = (result || []).filter(res => res.created || res.updated || res.deleted); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index bf2182dfcd..12fa811dc1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -80,7 +80,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements ngOnInit(): void { this.entityTypesVersionCreateFormGroup = this.fb.group({ - entityTypes: [this.fb.array([]), [Validators.min(1)]] + entityTypes: [this.fb.array([]), []] }); this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { this.updateModel(); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html new file mode 100644 index 0000000000..77cb887d8a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -0,0 +1,90 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.load-entities-relations' | translate }} + +
+
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss new file mode 100644 index 0000000000..b3e840198c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-load { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts new file mode 100644 index 0000000000..39709bebe1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-entity-types-version-load', + templateUrl: './entity-types-version-load.component.html', + styleUrls: ['./entity-types-version-load.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + } + ] +}) +export class EntityTypesVersionLoadComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionLoadConfig}; + + private propagateChange = null; + + public entityTypesVersionLoadFormGroup: FormGroup; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionLoadFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), []] + }); + this.entityTypesVersionLoadFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionLoadFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionLoadFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionLoadFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionLoadFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionLoadConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + loadRelations: [config.loadRelations, []], + removeOtherEntities: [config.removeOtherEntities, []] + }) + } + ); + return entityTypeControl; + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionLoadConfig = { + loadRelations: false, + removeOtherEntities: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + if (entityType) { + return this.translate.instant(entityTypeTranslations.get(entityType).typePlural); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionLoadConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 54f3e69302..b4541c7182 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -107,7 +107,8 @@ - +
@@ -125,6 +126,13 @@ (click)="toggleRestoreEntityVersion($event, restoreVersionButton, entityVersion)"> restore +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 8276e12d63..be714e6f97 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -48,6 +48,7 @@ import { MatButton } from '@angular/material/button'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @Component({ selector: 'tb-entity-versions-table', @@ -291,6 +292,27 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + toggleRestoreEntitiesVersion($event: Event, restoreEntitiesVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreEntitiesVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreEntitiesVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionLoadComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + onClose: (result: Array | null) => { + restoreEntitiesVersionPopover.hide(); + } + }, {}, {}, {}, false); + } + } + versionIdContent(entityVersion: EntityVersion): string { let versionId = entityVersion.id; if (versionId.length > 7) { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 9c5040f5f0..6e66daa24d 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -34,20 +34,11 @@ export interface VersionCreateConfig { saveRelations: boolean; } -export interface VersionLoadConfig { - loadRelations: boolean; -} - export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } -export enum VersionLoadRequestType { - SINGLE_ENTITY = 'SINGLE_ENTITY', - ENTITY_TYPE = 'ENTITY_TYPE' -} - export interface VersionCreateRequest { versionName: string; branch: string; @@ -97,6 +88,15 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: return res; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionLoadRequest { branch: string; versionId: string; @@ -109,6 +109,25 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { type: VersionLoadRequestType.SINGLE_ENTITY; } +export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { + removeOtherEntities: boolean; +} + +export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { + entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; + type: VersionLoadRequestType.ENTITY_TYPE; +} + +export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} { + const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + loadRelations: false, + removeOtherEntities: false + }; + } + return res; +} export interface BranchInfo { name: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5dfece08ed..49b4f82c17 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3144,12 +3144,18 @@ "sync-strategy-overwrite": "Overwrite", "entity-types": "Entity types", "sync-strategy": "Sync strategy", - "default": "Default", "all-entities": "All entities", "no-entity-types": "No entity types configured", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed." + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "load-entities-relations": "Load entities relations", + "remove-other-entities": "Remove other entities", + "restore-entities-from-version": "Restore entities from version '{{versionName}}'", + "no-entities-restored": "No entities restored", + "created": "{{created}} created", + "updated": "{{updated}} updated", + "deleted": "{{deleted}} deleted" }, "widget": { "widget-library": "Widgets Library", From bb9b58da5b13306bf32a2c0884e0d006f1b6e839 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 13:09:47 +0300 Subject: [PATCH 104/178] Add author to entity version --- .../DefaultEntitiesVersionControlService.java | 2 +- .../DefaultGitVersionControlQueueService.java | 35 +++++++++++++------ .../vc/GitVersionControlQueueService.java | 3 +- common/cluster-api/src/main/proto/queue.proto | 10 ++++-- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 6 ++-- .../sync/vc/DefaultGitRepositoryService.java | 12 +++++-- .../server/service/sync/vc/GitRepository.java | 7 ++-- .../server/service/sync/vc/PendingCommit.java | 8 ++++- .../vc/entity-versions-table.component.html | 8 ++++- .../vc/entity-versions-table.component.ts | 2 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 1 + 13 files changed, 71 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..0dc90fd109 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -120,7 +120,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings("UnstableApiUsage") @Override public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); + var pendingCommit = gitServiceQueue.prepareCommit(user, request); return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..891c68b802 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -66,10 +67,7 @@ import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -96,12 +94,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + public ListenableFuture prepareCommit(User user, VersionCreateRequest request) { SettableFuture future = SettableFuture.create(); - CommitGitRequest commit = new CommitGitRequest(tenantId, request); + CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); registerAndSend(commit, builder -> builder.setCommitRequest( - buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(request)).build() + buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() ).build(), wrap(future, commit)); return future; } @@ -356,7 +354,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -405,7 +403,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { @@ -453,8 +451,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return path; } - private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { - return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()) + .setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build(); + } + + private static String getAuthorName(User user) { + List parts = new ArrayList<>(); + if (StringUtils.isNotBlank(user.getFirstName())) { + parts.add(user.getFirstName()); + } + if (StringUtils.isNotBlank(user.getLastName())) { + parts.add(user.getLastName()); + } + if (parts.isEmpty()) { + parts.add(user.getName()); + } + return String.join(" ", parts); } private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..5a4ffe1706 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -36,7 +37,7 @@ import java.util.List; public interface GitVersionControlQueueService { - ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + ListenableFuture prepareCommit(User user, VersionCreateRequest request); ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 969ee0261c..3079978bce 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -692,14 +692,17 @@ message CommitResponseMsg { int64 ts = 1; string commitId = 2; string name = 3; - int32 added = 4; - int32 modified = 5; - int32 removed = 6; + string author = 4; + int32 added = 5; + int32 modified = 6; + int32 removed = 7; } message PrepareMsg { string commitMsg = 1; string branchName = 2; + string authorName = 3; + string authorEmail = 4; } message AddMsg { @@ -733,6 +736,7 @@ message EntityVersionProto { int64 ts = 1; string id = 2; string name = 3; + string author = 4; } message ListVersionsResponseMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 50d3fef8a8..c90336d3d7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -26,4 +26,5 @@ public class EntityVersion { private long timestamp; private String id; private String name; + private String author; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..dc42a61257 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -312,7 +312,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).setAuthor(v.getAuthor()).build() ).collect(Collectors.toList()))) ); } @@ -397,7 +397,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void prepareCommit(VersionControlRequestCtx ctx, UUID txId, PrepareMsg prepareMsg) { var tenantId = ctx.getTenantId(); - var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), prepareMsg.getCommitMsg()); + var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), + prepareMsg.getCommitMsg(), prepareMsg.getAuthorName(), prepareMsg.getAuthorEmail()); PendingCommit old = pendingCommitMap.get(tenantId); if (old != null) { doAbortCurrentCommit(tenantId, old); @@ -460,6 +461,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()) .setAdded(result.getAdded()) .setModified(result.getModified()) .setRemoved(result.getRemoved()))); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..8c30d3097a 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,7 +117,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName()); + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); repository.push(commit.getWorkingBranch(), commit.getBranch()); result.setVersion(toVersion(gitCommit)); @@ -255,7 +255,15 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage(), this.getAuthor(commit)); + } + + private String getAuthor(GitRepository.Commit commit) { + String author = String.format("<%s>", commit.getAuthorEmail()); + if (StringUtils.isNotBlank(commit.getAuthorName())) { + author = String.format("%s %s", commit.getAuthorName(), author); + } + return author; } public static EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..938094cfb8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -255,8 +255,9 @@ public class GitRepository { return new Status(status.getAdded(), modified, status.getRemoved()); } - public Commit commit(String message) throws GitAPIException { + public Commit commit(String message, String authorName, String authorEmail) throws GitAPIException { RevCommit revCommit = execute(git.commit() + .setAuthor(authorName, authorEmail) .setMessage(message)); return toCommit(revCommit); } @@ -325,7 +326,8 @@ public class GitRepository { } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), + revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); } private RevCommit resolveCommit(String id) throws IOException { @@ -470,6 +472,7 @@ public class GitRepository { private final String id; private final String message; private final String authorName; + private final String authorEmail; } @Data diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index b2eccd79db..ccd5fc685e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -30,12 +30,18 @@ public class PendingCommit { private String branch; private String versionName; - public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName) { + private String authorName; + + private String authorEmail; + + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { this.tenantId = tenantId; this.nodeId = nodeId; this.txId = txId; this.branch = branch; this.versionName = versionName; + this.authorName = authorName; + this.authorEmail = authorEmail; this.workingBranch = txId.toString(); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b4541c7182..d2dbc119bf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -101,11 +101,17 @@ - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} + + {{ 'version-control.author' | translate }} + + {{ entityVersion.author }} + + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index be714e6f97..589800e143 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -62,7 +62,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name', 'actions']; + displayedColumns = ['timestamp', 'id', 'name', 'author', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6e66daa24d..0ad52d2e0b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -138,6 +138,7 @@ export interface EntityVersion { timestamp: number; id: string; name: string; + author: string; } export interface VersionCreationResult { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 49b4f82c17..4a4e228cc2 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3119,6 +3119,7 @@ "create-entity-version": "Create entity version", "version-name": "Version name", "version-name-required": "Version name is required", + "author": "Author", "export-entity-relations": "Export entity relations", "entity-versions": "Entity versions", "versions": "Versions", From 07b6813465f958db07a9945f347a2f59b45af1d5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 13:19:16 +0300 Subject: [PATCH 105/178] Separate AutoCommit settings --- .../server/controller/AdminController.java | 72 +++++++++++-- .../DefaultEntitiesVersionControlService.java | 62 ++++++----- .../DefaultGitVersionControlQueueService.java | 12 +-- ...efaultTbVersionControlSettingsService.java | 102 ------------------ .../vc/EntitiesVersionControlService.java | 9 +- .../vc/GitVersionControlQueueService.java | 6 +- ...AbstractVersionControlSettingsService.java | 80 ++++++++++++++ .../AutoCommitSettingsCaffeineCache.java} | 14 ++- .../AutoCommitSettingsRedisCache.java} | 17 ++- .../DefaultTbAutoCommitSettingsService.java | 36 +++++++ .../TbAutoCommitSettingsService.java | 30 ++++++ .../DefaultTbRepositorySettingsService.java | 63 +++++++++++ .../RepositorySettingsCaffeineCache.java | 34 ++++++ .../RepositorySettingsRedisCache.java | 36 +++++++ .../TbRepositorySettingsService.java} | 12 +-- .../src/main/resources/thingsboard.yml | 9 +- .../server/common/data/CacheConstants.java | 3 +- .../data/sync/vc/AutoCommitSettings.java | 27 +++++ ...lSettings.java => RepositorySettings.java} | 11 +- .../DefaultClusterVersionControlService.java | 6 +- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../server/service/sync/vc/GitRepository.java | 12 +-- .../service/sync/vc/GitRepositoryService.java | 8 +- .../sync/vc/VersionControlRequestCtx.java | 6 +- 24 files changed, 467 insertions(+), 208 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsCaffeineCache.java => autocommit/AutoCommitSettingsCaffeineCache.java} (66%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsRedisCache.java => autocommit/AutoCommitSettingsRedisCache.java} (58%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{TbVersionControlSettingsService.java => repository/TbRepositorySettingsService.java} (60%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{EntitiesVersionControlSettings.java => RepositorySettings.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index cbc6ff2bca..67665ad8c7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -29,24 +29,23 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.*; -import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -68,6 +67,9 @@ public class AdminController extends BaseController { @Autowired private EntitiesVersionControlService versionControlService; + @Autowired + private TbAutoCommitSettingsService autoCommitSettingsService; + @Autowired private UpdateService updateService; @@ -194,10 +196,10 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vcSettings") @ResponseBody - public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getVersionControlSettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); versionControlSettings.setPassword(null); versionControlSettings.setPrivateKey(null); versionControlSettings.setPrivateKeyPassword(null); @@ -225,9 +227,9 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); + ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { savedSettings.setPassword(null); savedSettings.setPrivateKey(null); @@ -251,13 +253,65 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", + notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings") + @ResponseBody + public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return checkNotNull(autoCommitSettingsService.get(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings/exists") + @ResponseBody + public Boolean autoCommitSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return autoCommitSettingsService.get(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vc/autoCommitSettings") + public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + return autoCommitSettingsService.save(getTenantId(), settings); + } + + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + autoCommitSettingsService.delete(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) public DeferredResult checkVersionControlAccess( @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + @RequestBody RepositorySettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); settings = checkNotNull(settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..3060b05b1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -27,7 +27,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -44,7 +43,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -62,19 +61,18 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -94,7 +92,8 @@ import static com.google.common.util.concurrent.Futures.transformAsync; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final TbVersionControlSettingsService vcSettingsService; + private final TbRepositorySettingsService repositorySettingsService; + private final TbAutoCommitSettingsService autoCommitSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; @@ -336,16 +335,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - return vcSettingsService.get(tenantId); + public RepositorySettings getVersionControlSettings(TenantId tenantId) { + return repositorySettingsService.get(tenantId); } @Override - public ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - var restoredSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); + public ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) { + var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings); try { var future = gitServiceQueue.initRepository(tenantId, restoredSettings); - return Futures.transform(future, f -> vcSettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); + return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); } catch (Exception e) { log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); throw new RuntimeException("Failed to init repository!", e); @@ -354,7 +353,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (vcSettingsService.delete(tenantId)) { + if (repositorySettingsService.delete(tenantId)) { return gitServiceQueue.clearRepository(tenantId); } else { return Futures.immediateFuture(null); @@ -362,8 +361,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - settings = this.vcSettingsService.restore(tenantId, settings); + public ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException { + settings = this.repositorySettingsService.restore(tenantId, settings); try { return gitServiceQueue.testRepository(tenantId, settings); } catch (Exception e) { @@ -374,22 +373,29 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { - var settings = vcSettingsService.get(user.getTenantId()); + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } var entityType = entityId.getEntityType(); - if (settings != null && settings.getAutoCommitSettings() != null && settings.getAutoCommitSettings().containsKey(entityType)) { - AutoVersionCreateConfig autoCommitConfig = settings.getAutoCommitSettings().get(entityType); - SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); - var autoCommitBranchName = autoCommitConfig.getBranch(); - if (StringUtils.isEmpty(autoCommitBranchName)) { - autoCommitBranchName = StringUtils.isNotEmpty(settings.getDefaultBranch()) ? settings.getDefaultBranch() : "auto-commits"; - } - vcr.setBranch(autoCommitBranchName); - vcr.setVersionName("auto-commit by " + user.getEmail() + " at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); - vcr.setEntityId(entityId); - vcr.setConfig(autoCommitConfig); - return saveEntitiesVersion(user, vcr); + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; } - return Futures.immediateFuture(null); + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); } private String getCauseMessage(Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..7f501522af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -271,7 +271,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private void registerAndSend(PendingGitRequest request, - Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { + Function enrichFunction, RepositorySettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); var requestBody = enrichFunction.apply(newRequestProto(request, settings)); @@ -307,7 +307,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() @@ -317,7 +317,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder @@ -457,7 +457,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); } - private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, RepositorySettings settings) { var tenantId = request.getTenantId(); var requestId = request.getRequestId(); var builder = ToVersionControlServiceMsg.newBuilder() @@ -466,7 +466,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()); - EntitiesVersionControlSettings vcSettings = settings; + RepositorySettings vcSettings = settings; if (vcSettings == null && request.requiresSettings()) { vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java deleted file mode 100644 index 1d1fff81a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { - - public static final String SETTINGS_KEY = "entitiesVersionControl"; - private final AdminSettingsService adminSettingsService; - private final TbTransactionalCache cache; - - @Override - public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { - EntitiesVersionControlSettings storedSettings = get(tenantId); - if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } - - @Override - public EntitiesVersionControlSettings get(TenantId tenantId) { - EntitiesVersionControlSettings settings = cache.getAndPutInTransaction(tenantId, () -> { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - }, true); - if (settings != null) { - settings = new EntitiesVersionControlSettings(settings); - } - return settings; - } - - @Override - public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. - cache.evict(tenantId); - return savedVersionControlSettings; - } - - @Override - public boolean delete(TenantId tenantId) { - boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); - cache.evict(tenantId); - return result; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 836a75e021..447ae1d021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -24,7 +23,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -54,13 +53,13 @@ public interface EntitiesVersionControlService { ListenableFuture> listBranches(TenantId tenantId) throws Exception; - EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + RepositorySettings getVersionControlSettings(TenantId tenantId); - ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings); ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception; - ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..221206f5bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -64,9 +64,9 @@ public interface GitVersionControlQueueService { ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); - ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); - ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); ListenableFuture clearRepository(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java new file mode 100644 index 0000000000..e826c32c56 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.io.Serializable; + +public abstract class TbAbstractVersionControlSettingsService { + + private final String settingsKey; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + private final Class clazz; + + public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache, Class clazz, String settingsKey) { + this.adminSettingsService = adminSettingsService; + this.cache = cache; + this.clazz = clazz; + this.settingsKey = settingsKey; + } + + public T get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load " + settingsKey + " settings!", e); + } + } + return null; + }, true); + } + + public T save(TenantId tenantId, T settings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(settingsKey); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + T savedSettings; + try { + savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load auto commit settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedSettings; + } + + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, settingsKey); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java index 8cab03ca29..47b9e5f5d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java @@ -13,24 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CaffeineTbTransactionalCache; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) -@Service("VersionControlCache") -public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache { - public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { - super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java index 9c778d6e4f..f88b1cf6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -13,29 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; -import com.google.protobuf.InvalidProtocolBufferException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.RedisSerializer; -import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") -@Service("VersionControlCache") -public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { - public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java new file mode 100644 index 0000000000..b6e8d45ec2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService implements TbAutoCommitSettingsService { + + public static final String SETTINGS_KEY = "autoCommitSettings"; + + public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java new file mode 100644 index 0000000000..51978481e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbAutoCommitSettingsService { + + AutoCommitSettings get(TenantId tenantId); + + AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java new file mode 100644 index 0000000000..608efec870 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbRepositorySettingsService extends TbAbstractVersionControlSettingsService implements TbRepositorySettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + + public DefaultTbRepositorySettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, RepositorySettings.class, SETTINGS_KEY); + } + + @Override + public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { + RepositorySettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public RepositorySettings get(TenantId tenantId) { + RepositorySettings settings = super.get(tenantId); + if (settings != null) { + settings = new RepositorySettings(settings); + } + return settings; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java new file mode 100644 index 0000000000..60b7f50e12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RepositorySettingsCache") +public class RepositorySettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public RepositorySettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.REPOSITORY_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java new file mode 100644 index 0000000000..3cccb6d24e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RepositorySettingsCache") +public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { + + public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java index 178499c6c7..946d06c87c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.repository; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -public interface TbVersionControlSettingsService { +public interface TbRepositorySettingsService { - EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings restore(TenantId tenantId, RepositorySettings versionControlSettings); - EntitiesVersionControlSettings get(TenantId tenantId); + RepositorySettings get(TenantId tenantId); - EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings save(TenantId tenantId, RepositorySettings versionControlSettings); boolean delete(TenantId tenantId); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 31e5d7a76e..9663396d7e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,9 +433,12 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" - vcSettings: - timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" + repositorySettings: + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + autoCommitSettings: + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 85ea116f0a..94ca9db5a5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,5 +31,6 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; - public static final String VC_SETTINGS_CACHE = "vcSettings"; + public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings"; + public static final String AUTO_COMMIT_SETTINGS_CACHE = "autoCommitSettings"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java new file mode 100644 index 0000000000..2120ec4c44 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; + +import java.util.HashMap; + +public class AutoCommitSettings extends HashMap { + + private static final long serialVersionUID = -5757067601838792059L; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java similarity index 80% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index 7764274d22..c576f82298 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -24,7 +25,8 @@ import java.util.HashMap; import java.util.Map; @Data -public class EntitiesVersionControlSettings implements Serializable { +@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. +public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; @@ -36,12 +38,10 @@ public class EntitiesVersionControlSettings implements Serializable { private String privateKeyPassword; private String defaultBranch; - private Map autoCommitSettings; - - public EntitiesVersionControlSettings() { + public RepositorySettings() { } - public EntitiesVersionControlSettings(EntitiesVersionControlSettings settings) { + public RepositorySettings(RepositorySettings settings) { this.repositoryUri = settings.getRepositoryUri(); this.authMethod = settings.getAuthMethod(); this.username = settings.getUsername(); @@ -50,6 +50,5 @@ public class EntitiesVersionControlSettings implements Serializable { this.privateKey = settings.getPrivateKey(); this.privateKeyPassword = settings.getPrivateKeyPassword(); this.defaultBranch = settings.getDefaultBranch(); - this.autoCommitSettings = settings.getAutoCommitSettings() != null ? new HashMap<>(settings.getAutoCommitSettings()) : new HashMap<>(); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..9463608b76 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -495,8 +495,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } - private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { - Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + private RepositorySettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); if (settingsOpt.isPresent()) { return settingsOpt.get(); } else { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..b12e03e762 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -216,13 +216,13 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository.test(settings, repositoryDirectory.toFile()); } @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception { clearRepository(tenantId); log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); @@ -238,7 +238,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception { + public RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception { var gitRepository = repositories.get(tenantId); return gitRepository != null ? gitRepository.getSettings() : null; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..b5bc6355d4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -58,7 +58,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; @@ -77,14 +77,14 @@ public class GitRepository { private final Git git; @Getter - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; private final CredentialsProvider credentialsProvider; private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, EntitiesVersionControlSettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.settings = settings; this.credentialsProvider = credentialsProvider; @@ -92,7 +92,7 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { @@ -109,7 +109,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { + public static GitRepository open(File directory, RepositorySettings settings) throws IOException { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; @@ -121,7 +121,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index ec69e0c80e..057fa9c1ab 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -19,7 +19,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -39,11 +39,11 @@ public interface GitRepositoryService { List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception; + RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception; void clearRepository(TenantId tenantId) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java index 6541a2978a..26ab0d94c1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.sync.vc; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import java.util.UUID; @@ -29,9 +29,9 @@ public class VersionControlRequestCtx { private final String nodeId; private final UUID requestId; private final TenantId tenantId; - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; - public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, RepositorySettings settings) { this.nodeId = msg.getNodeId(); this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); From 32f2f4ccd07c044b9396c42b556146c4daa3c832 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 13:19:36 +0300 Subject: [PATCH 106/178] Export/import of attributes --- .../impl/DefaultEntityExportService.java | 79 ++++++++++++--- .../impl/BaseEntityImportService.java | 97 +++++++++++++++---- .../DefaultEntitiesVersionControlService.java | 15 ++- .../data/sync/ie/AttributeExportData.java | 30 ++++++ .../common/data/sync/ie/EntityExportData.java | 2 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 9 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 31bf768e90..8e723f8793 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -19,22 +19,31 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; @Service @TbCoreComponent @@ -45,6 +54,8 @@ public class DefaultEntityExportService relations = new ArrayList<>(); + List relations = exportRelations(user, entity); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(user, entity); + exportData.setAttributes(attributes); + } + } - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - relations.addAll(inboundRelations); + private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - relations.addAll(outboundRelations); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + relations.addAll(inboundRelations); - exportData.setRelations(relations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES); + + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new HashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 203b0caa16..caa1e20ad7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -29,23 +32,36 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; +@Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -53,6 +69,8 @@ public abstract class BaseEntityImportService { - if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { - return; - } - - List relations = new ArrayList<>(exportData.getRelations()); + if (importSettings.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(user, exportData.getRelations(), importResult); + } + if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) { + importAttributes(user, exportData.getAttributes(), importResult); + } + } + private void importRelations(SecurityUser user, List relations, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { - if (!relation.getTo().equals(savedEntity.getId())) { + if (!relation.getTo().equals(entity.getId())) { HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { + if (!relation.getFrom().equals(entity.getId())) { HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); } } - if (oldEntity != null) { + if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { EntityId otherEntity = null; - if (!existingRelation.getTo().equals(savedEntity.getId())) { + if (!existingRelation.getTo().equals(entity.getId())) { otherEntity = existingRelation.getTo(); - } else if (!existingRelation.getFrom().equals(savedEntity.getId())) { + } else if (!existingRelation.getFrom().equals(entity.getId())) { otherEntity = existingRelation.getFrom(); } if (otherEntity != null) { @@ -161,6 +183,44 @@ public abstract class BaseEntityImportService> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) {} + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), @@ -168,6 +228,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) @@ -181,6 +242,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) @@ -216,7 +278,8 @@ public abstract class BaseEntityImportService saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -208,6 +208,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(false) .build(), true, true); } catch (Exception e) { @@ -246,6 +247,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(config.isFindExistingEntityByName()) .build(), false, false); @@ -283,6 +285,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); VersionLoadResult result = results.get(entityType); result.setDeleted(result.getDeleted() + 1); } @@ -321,6 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(true) + .exportAttributes(true) .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 35fceafe43..542e675940 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; +import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @@ -45,5 +46,6 @@ public class EntityExportData> { private EntityType entityType; private List relations; + private Map> attributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 051745c07f..0800a1f7c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportRelations; + private boolean exportAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 3e18bc9f5f..564b0134e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 10971f1170..4426717fe9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -24,4 +24,5 @@ public class VersionCreateConfig implements Serializable { private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 8ed9c34694..2d68a2b7e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,5 +21,6 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; + private boolean loadAttributes; } From 09402f40acb56b7f0d14ade095c4c509cabebd36 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 16:16:24 +0300 Subject: [PATCH 107/178] UI: Refactor vc settings to repository settings. Add remove other entities confirm. --- .../server/controller/AdminController.java | 77 ++++--- .../DefaultTbRepositorySettingsService.java | 8 +- ...hMethod.java => RepositoryAuthMethod.java} | 2 +- .../data/sync/vc/RepositorySettings.java | 6 +- .../server/service/sync/vc/GitRepository.java | 14 +- ui-ngx/src/app/core/auth/auth.actions.ts | 10 +- ui-ngx/src/app/core/auth/auth.models.ts | 2 +- ui-ngx/src/app/core/auth/auth.reducer.ts | 4 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 4 +- ui-ngx/src/app/core/auth/auth.service.ts | 10 +- ui-ngx/src/app/core/http/admin.service.ts | 47 ++-- .../http/entities-version-control.service.ts | 8 +- ui-ngx/src/app/core/services/menu.service.ts | 8 +- .../home/components/home-components.module.ts | 13 +- ...entity-types-version-create.component.html | 13 +- .../entity-types-version-create.component.ts | 2 + .../entity-types-version-load.component.html | 28 ++- .../vc/entity-types-version-load.component.ts | 42 +++- .../vc/entity-version-create.component.html | 3 + .../vc/entity-version-create.component.ts | 6 +- .../vc/entity-version-restore.component.html | 3 + .../vc/entity-version-restore.component.ts | 6 +- ...move-other-entities-confirm.component.html | 41 ++++ ...remove-other-entities-confirm.component.ts | 66 ++++++ ...tml => repository-settings.component.html} | 26 +-- ...css => repository-settings.component.scss} | 2 +- .../vc/repository-settings.component.ts | 216 +++++++++++++++++ .../vc/version-control-settings.component.ts | 218 ------------------ .../vc/version-control.component.html | 6 +- .../vc/version-control.component.ts | 10 +- .../home/pages/admin/admin-routing.module.ts | 10 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ... repository-admin-settings.component.html} | 2 +- ...=> repository-admin-settings.component.ts} | 12 +- ui-ngx/src/app/shared/models/constants.ts | 2 +- .../src/app/shared/models/settings.models.ts | 19 +- ui-ngx/src/app/shared/models/vc.models.ts | 23 +- .../assets/locale/locale.constant-en_US.json | 19 +- 38 files changed, 592 insertions(+), 400 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{VersionControlAuthMethod.java => RepositoryAuthMethod.java} (94%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.html => repository-settings.component.html} (78%) rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.scss => repository-settings.component.scss} (96%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts delete mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.html => repository-admin-settings.component.html} (87%) rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.ts => repository-admin-settings.component.ts} (68%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 67665ad8c7..ed740f73fb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -191,12 +191,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Get version control settings (getVersionControlSettings)", - notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Get repository settings (getRepositorySettings)", + notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings") + @GetMapping("/repositorySettings") @ResponseBody - public RepositorySettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); @@ -209,12 +209,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check repository settings exists (repositorySettingsExists)", + notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings/exists") + @GetMapping("/repositorySettings/exists") @ResponseBody - public Boolean versionControlSettingsExists() throws ThingsboardException { + public Boolean repositorySettingsExists() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); return versionControlService.getVersionControlSettings(getTenantId()) != null; @@ -223,11 +223,11 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)", + notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { + @PostMapping("/repositorySettings") + public DeferredResult saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { @@ -238,13 +238,13 @@ public class AdminController extends BaseController { }, MoreExecutors.directExecutor())); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete repository settings (deleteRepositorySettings)", + notes = "Deletes the repository settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) - public DeferredResult deleteVersionControlSettings() throws ThingsboardException { + public DeferredResult deleteRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId())); @@ -253,6 +253,23 @@ public class AdminController extends BaseController { } } + + @ApiOperation(value = "Check repository access (checkRepositoryAccess)", + notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST) + public DeferredResult checkRepositoryAccess( + @ApiParam(value = "A JSON value representing the Repository Settings.") + @RequestBody RepositorySettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -267,8 +284,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", + notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vc/autoCommitSettings/exists") @ResponseBody @@ -281,8 +298,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", + notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vc/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { @@ -290,8 +307,8 @@ public class AdminController extends BaseController { return autoCommitSettingsService.save(getTenantId(), settings); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)", + notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) @@ -305,22 +322,6 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control access (checkVersionControlAccess)", - notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) - public DeferredResult checkVersionControlAccess( - @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody RepositorySettings settings) throws ThingsboardException { - try { - accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - settings = checkNotNull(settings); - return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); - } catch (Exception e) { - throw handleException(e); - } - } - @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java index 608efec870..555490bd06 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -19,7 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; @@ -38,10 +38,10 @@ public class DefaultTbRepositorySettingsService extends TbAbstractVersionControl public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { RepositorySettings storedSettings = get(tenantId); if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + RepositoryAuthMethod authMethod = settings.getAuthMethod(); + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { settings.setPrivateKey(storedSettings.getPrivateKey()); if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java index 770a1e582c..3c0e5849a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; -public enum VersionControlAuthMethod { +public enum RepositoryAuthMethod { USERNAME_PASSWORD, PRIVATE_KEY } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index c576f82298..eb1a97ff6c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -17,12 +17,8 @@ package org.thingsboard.server.common.data.sync.vc; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; @Data @JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. @@ -30,7 +26,7 @@ public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; - private VersionControlAuthMethod authMethod; + private RepositoryAuthMethod authMethod; private String username; private String password; private String privateKeyFileName; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dd9a67aed4..2b2802d003 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -59,7 +59,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -95,9 +95,9 @@ public class GitRepository { public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } CloneCommand cloneCommand = Git.cloneRepository() @@ -113,9 +113,9 @@ public class GitRepository { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); @@ -124,9 +124,9 @@ public class GitRepository { public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 872607b822..a60720e61d 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -24,7 +24,7 @@ export enum AuthActionTypes { LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', - UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' + UPDATE_HAS_REPOSITORY = '[Auth] Change Has Repository' } export class ActionAuthAuthenticated implements Action { @@ -55,11 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } -export class ActionAuthUpdateHasVersionControl implements Action { - readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; +export class ActionAuthUpdateHasRepository implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_REPOSITORY; - constructor(readonly payload: { hasVersionControl: boolean }) {} + constructor(readonly payload: { hasRepository: boolean }) {} } export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasRepository; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index a69b783526..33b84945aa 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,7 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; - hasVersionControl: boolean; + hasRepository: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 9d9404524b..cace62660e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -24,7 +24,7 @@ const emptyUserAuthState: AuthPayload = { forceFullscreen: false, allowedDashboardIds: [], edgesSupportEnabled: false, - hasVersionControl: false + hasRepository: false }; export const initialState: AuthState = { @@ -55,7 +55,7 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; - case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + case AuthActionTypes.UPDATE_HAS_REPOSITORY: return { ...state, ...action.payload}; default: diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index 7a099d5acd..4a63dbcbed 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,9 +55,9 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); -export const selectHasVersionControl = createSelector( +export const selectHasRepository = createSelector( selectAuthState, - (state: AuthState) => state.hasVersionControl + (state: AuthState) => state.hasRepository ); export function getCurrentAuthState(store: Store): AuthState { diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6fe809955f..74f6689553 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,9 +437,9 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } - private loadHasVersionControl(authUser: AuthUser): Observable { + private loadHasRepository(authUser: AuthUser): Observable { if (authUser.authority === Authority.TENANT_ADMIN) { - return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + return this.http.get('/api/admin/repositorySettings/exists', defaultHttpOptions()); } else { return of(false); } @@ -449,15 +449,15 @@ export class AuthService { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), - this.loadHasVersionControl(authPayload.authUser), + this.loadHasRepository(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - const hasVersionControl: boolean = data[3] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; + const hasRepository: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasRepository}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 7c12bd3a5d..38f18bd598 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -15,19 +15,21 @@ /// import { Injectable } from '@angular/core'; -import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, - EntitiesVersionControlSettings, + RepositorySettings, MailServerSettings, SecuritySettings, TestSmsRequest, - UpdateMessage + UpdateMessage, AutoCommitSettings } from '@shared/models/settings.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { tap } from 'rxjs/operators'; +import { AuthUser } from '@shared/models/user.model'; +import { Authority } from '@shared/models/authority.enum'; @Injectable({ providedIn: 'root' @@ -68,13 +70,13 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } - public getEntitiesVersionControlSettings(config?: RequestConfig): Observable { - return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + public getRepositorySettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); } - public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings', versionControlSettings, + public saveRepositorySettings(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings', repositorySettings, defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); @@ -82,17 +84,34 @@ export class AdminService { ); } - public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + public deleteRepositorySettings(config?: RequestConfig) { + return this.http.delete('/api/admin/repositorySettings', defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); }) ); } - public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + public checkRepositoryAccess(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings/checkAccess', repositorySettings, defaultHttpOptionsFromConfig(config)); + } + + public getAutoCommitSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); + } + + private autoCommitSettingsExists(config?: RequestConfig): Observable { + return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); + } + + public saveAutoCommitSettings(autoCommitSettings: AutoCommitSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/autoCommitSettings', autoCommitSettings, defaultHttpOptionsFromConfig(config)); + } + + public deleteAutoCommitSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/autoCommitSettings', defaultHttpOptionsFromConfig(config)); } public checkUpdates(config?: RequestConfig): Observable { diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 7578696774..9f6f4f3403 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,7 +17,7 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { combineLatest, Observable, of } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { BranchInfo, EntityDataDiff, EntityVersion, @@ -29,10 +29,10 @@ import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; -import { createSelector, select, Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; -import { catchError, combineAll, tap } from 'rxjs/operators'; +import { selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index ee81f007ee..50938e5869 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -388,9 +388,9 @@ export class MenuService { }, { id: guid(), - name: 'admin.git-settings', + name: 'admin.repository-settings', type: 'link', - path: '/settings/vc', + path: '/settings/repository', icon: 'manage_history' } ] @@ -538,9 +538,9 @@ export class MenuService { path: '/settings/resources-library' }, { - name: 'admin.git-settings', + name: 'admin.repository-settings', icon: 'manage_history', - path: '/settings/vc', + path: '/settings/repository', } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index e9fc3f8c9a..613f236cf6 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -153,7 +153,7 @@ import { TenantProfileQueuesComponent } from '@home/components/profile/queue/ten import { QueueFormComponent } from '@home/components/queue/queue-form.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; @@ -163,6 +163,7 @@ import { ComplexVersionCreateComponent } from '@home/components/vc/complex-versi import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @NgModule({ declarations: @@ -287,7 +288,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -296,7 +297,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], imports: [ CommonModule, @@ -415,7 +417,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -424,7 +426,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 9d1192c0c9..3c1dd73117 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -53,7 +53,7 @@ [allowedEntityTypes]="allowedEntityTypes(entityTypeFormGroup)">
- + version-control.sync-strategy @@ -64,9 +64,14 @@ - - {{ 'version-control.export-entity-relations' | translate }} - +
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
-
+
-
- - {{ 'version-control.remove-other-entities' | translate }} - - - {{ 'version-control.load-entities-relations' | translate }} - +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} + +
+
+ + {{ 'version-control.load-entities-relations' | translate }} + + + {{ 'version-control.load-entities-attributes' | translate }} + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 39709bebe1..40bf9ad869 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { Component, forwardRef, Input, OnInit, Renderer2, ViewContainerRef } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -28,11 +28,15 @@ import { Validators } from '@angular/forms'; import { PageComponent } from '@shared/components/page.component'; -import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes, VersionCreationResult } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { TranslateService } from '@ngx-translate/core'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { MatCheckbox } from '@angular/material/checkbox/checkbox'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @Component({ selector: 'tb-entity-types-version-load', @@ -64,6 +68,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On constructor(protected store: Store, private translate: TranslateService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private viewContainerRef: ViewContainerRef, private fb: FormBuilder) { super(store); } @@ -124,7 +131,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On entityType: [entityType, [Validators.required]], config: this.fb.group({ loadRelations: [config.loadRelations, []], - removeOtherEntities: [config.removeOtherEntities, []] + loadAttributes: [config.loadAttributes, []], + removeOtherEntities: [config.removeOtherEntities, []], + findExistingEntityByName: [config.findExistingEntityByName, []] }) } ); @@ -156,7 +165,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -194,6 +205,29 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On return res; } + onRemoveOtherEntities(removeOtherEntitiesCheckbox: MatCheckbox, entityTypeControl: AbstractControl, $event: Event) { + const removeOtherEntities: boolean = entityTypeControl.get('config.removeOtherEntities').value; + if (!removeOtherEntities) { + $event.preventDefault(); + $event.stopPropagation(); + const trigger = $('.mat-checkbox-frame', removeOtherEntitiesCheckbox._elementRef.nativeElement)[0]; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const removeOtherEntitiesConfirmPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, RemoveOtherEntitiesConfirmComponent, 'bottom', true, null, + { + onClose: (result: boolean | null) => { + removeOtherEntitiesConfirmPopover.hide(); + if (result) { + entityTypeControl.get('config').get('removeOtherEntities').patchValue(true, {emitEvent: true}); + } + } + }, {}, {}, {}, false); + } + } + } + private updateModel() { const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 5c575b93d4..b5f2fb6937 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -41,6 +41,9 @@ {{ 'version-control.export-entity-relations' | translate }} + + {{ 'version-control.export-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 9f998bc2af..2ff8c03227 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -62,7 +62,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], versionName: [null, [Validators.required]], - saveRelations: [false, []] + saveRelations: [false, []], + saveAttributes: [false, []] }); } @@ -78,7 +79,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni branch: this.createVersionFormGroup.get('branch').value, versionName: this.createVersionFormGroup.get('versionName').value, config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index cb71696b4e..e84beb03dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -29,6 +29,9 @@ {{ 'version-control.load-entity-relations' | translate }} + + {{ 'version-control.load-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 2cc615c011..ea5443e036 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -57,7 +57,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [false, []] + loadRelations: [false, []], + loadAttributes: [false, []] }); } @@ -73,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn versionId: this.versionId, externalEntityId: this.externalEntityId, config: { - loadRelations: this.restoreFormGroup.get('loadRelations').value + loadRelations: this.restoreFormGroup.get('loadRelations').value, + loadAttributes: this.restoreFormGroup.get('loadAttributes').value }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html new file mode 100644 index 0000000000..14ae91e2f2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html @@ -0,0 +1,41 @@ + +
+
+
+
+ + + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts new file mode 100644 index 0000000000..f569d96fac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts @@ -0,0 +1,66 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-remove-other-entities-confirm', + templateUrl: './remove-other-entities-confirm.component.html', + styleUrls: [] +}) +export class RemoveOtherEntitiesConfirmComponent extends PageComponent implements OnInit { + + @Input() + onClose: (result: boolean | null) => void; + + confirmFormGroup: FormGroup; + + removeOtherEntitiesConfirmText: SafeHtml; + + removeOtherEntitiesVerificationText = 'remove other entities'; + + constructor(protected store: Store, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + this.removeOtherEntitiesConfirmText = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.remove-other-entities-confirm-text')); + } + + ngOnInit(): void { + this.confirmFormGroup = this.fb.group({ + verification: [null, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + confirm(): void { + if (this.onClose) { + this.onClose(true); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html similarity index 78% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index b48162d06f..bdab6c061b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,24 +16,24 @@ -->
- +
- admin.git-repository-settings + admin.repository-settings -
+
-
+
admin.repository-url - + admin.repository-url-required @@ -46,12 +46,12 @@ admin.auth-method - - {{versionControlAuthMethodTranslations.get(method) | translate}} + + {{repositoryAuthMethodTranslations.get(method) | translate}} -
+
common.username
-
+
+ (fileNameChanged)="repositorySettingsForm.get('privateKeyFileName').patchValue($event)"> @@ -96,10 +96,10 @@ -
diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss index d1d55faf19..77ae0ffd8f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - mat-card.vc-settings { + mat-card.repository-settings { margin: 8px; } .fields-group { diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts new file mode 100644 index 0000000000..c73cf0e632 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -0,0 +1,216 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + RepositorySettings, + RepositoryAuthMethod, + repositoryAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; +import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { catchError, mergeMap, take } from 'rxjs/operators'; +import { of } from 'rxjs'; + +@Component({ + selector: 'tb-repository-settings', + templateUrl: './repository-settings.component.html', + styleUrls: ['./repository-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class RepositorySettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; + + repositorySettingsForm: FormGroup; + settings: RepositorySettings = null; + + repositoryAuthMethod = RepositoryAuthMethod; + repositoryAuthMethods = Object.values(RepositoryAuthMethod); + repositoryAuthMethodTranslations = repositoryAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.repositorySettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: ['main', []], + authMethod: [RepositoryAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.repositorySettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.repositorySettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.store.pipe( + select(selectHasRepository), + take(1), + mergeMap((hasRepository) => { + if (hasRepository) { + return this.adminService.getRepositorySettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + if (this.settings != null) { + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + } + }); + } + + checkAccess(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.checkRepositoryAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-repository-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.saveRepositorySettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: true })); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-repository-settings-title', ), + this.translate.instant('admin.delete-repository-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteRepositorySettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.repositorySettingsForm.reset({ defaultBranch: 'main', authMethod: RepositoryAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: false })); + } + ); + } + }); + } + + changePasswordChanged() { + if (this.changePassword) { + this.repositorySettingsForm.get('password').patchValue(''); + this.repositorySettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').patchValue(''); + this.repositorySettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: RepositoryAuthMethod = this.repositorySettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.repositorySettingsForm.get('privateKeyFileName').value; + if (authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.repositorySettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.repositorySettingsForm.get('password').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('password').disable({emitEvent}); + } + this.repositorySettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.repositorySettingsForm.get('privateKey').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.repositorySettingsForm.get('username').disable({emitEvent}); + this.repositorySettingsForm.get('password').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.repositorySettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.repositorySettingsForm.get('privateKey').clearValidators(); + } else { + this.repositorySettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.repositorySettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts deleted file mode 100644 index 3462fbb728..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ /dev/null @@ -1,218 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Input, OnInit } from '@angular/core'; -import { PageComponent } from '@shared/components/page.component'; -import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; -import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { select, Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { AdminService } from '@core/http/admin.service'; -import { - EntitiesVersionControlSettings, - VersionControlAuthMethod, - versionControlAuthMethodTranslationMap -} from '@shared/models/settings.models'; -import { ActionNotificationShow } from '@core/notification/notification.actions'; -import { TranslateService } from '@ngx-translate/core'; -import { isNotEmptyStr } from '@core/utils'; -import { DialogService } from '@core/services/dialog.service'; -import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; -import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; -import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap, take } from 'rxjs/operators'; -import { of } from 'rxjs'; - -@Component({ - selector: 'tb-version-control-settings', - templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] -}) -export class VersionControlSettingsComponent extends PageComponent implements OnInit { - - @Input() - detailsMode = false; - - versionControlSettingsForm: FormGroup; - settings: EntitiesVersionControlSettings = null; - - versionControlAuthMethod = VersionControlAuthMethod; - versionControlAuthMethods = Object.values(VersionControlAuthMethod); - versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; - - showChangePassword = false; - changePassword = false; - - showChangePrivateKeyPassword = false; - changePrivateKeyPassword = false; - - constructor(protected store: Store, - private adminService: AdminService, - private dialogService: DialogService, - private translate: TranslateService, - public fb: FormBuilder) { - super(store); - } - - ngOnInit() { - this.versionControlSettingsForm = this.fb.group({ - repositoryUri: [null, [Validators.required]], - defaultBranch: ['main', []], - authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], - username: [null, []], - password: [null, []], - privateKeyFileName: [null, [Validators.required]], - privateKey: [null, []], - privateKeyPassword: [null, []] - }); - this.updateValidators(false); - this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { - this.updateValidators(true); - }); - this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { - this.updateValidators(false); - }); - this.store.pipe( - select(selectHasVersionControl), - take(1), - mergeMap((hasVersionControl) => { - if (hasVersionControl) { - return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( - catchError(() => of(null)) - ); - } else { - return of(null); - } - }) - ).subscribe( - (settings) => { - this.settings = settings; - if (this.settings != null) { - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - } - }); - } - - checkAccess(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.checkVersionControlAccess(settings).subscribe(() => { - this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), - type: 'success' })); - }); - } - - save(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( - (savedSettings) => { - this.settings = savedSettings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - this.changePassword = false; - } else { - this.showChangePrivateKeyPassword = true; - this.changePrivateKeyPassword = false; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); - } - ); - } - - delete(formDirective: FormGroupDirective): void { - this.dialogService.confirm( - this.translate.instant('admin.delete-git-settings-title', ), - this.translate.instant('admin.delete-git-settings-text'), null, - this.translate.instant('action.delete') - ).subscribe((data) => { - if (data) { - this.adminService.deleteEntitiesVersionControlSettings().subscribe( - () => { - this.settings = null; - this.showChangePassword = false; - this.changePassword = false; - this.showChangePrivateKeyPassword = false; - this.changePrivateKeyPassword = false; - formDirective.resetForm(); - this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); - } - ); - } - }); - } - - changePasswordChanged() { - if (this.changePassword) { - this.versionControlSettingsForm.get('password').patchValue(''); - this.versionControlSettingsForm.get('password').markAsDirty(); - } - this.updateValidators(false); - } - - changePrivateKeyPasswordChanged() { - if (this.changePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); - this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); - } - this.updateValidators(false); - } - - updateValidators(emitEvent?: boolean): void { - const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; - const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; - if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.versionControlSettingsForm.get('username').enable({emitEvent}); - if (this.changePassword || !this.showChangePassword) { - this.versionControlSettingsForm.get('password').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('password').disable({emitEvent}); - } - this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } else { - this.versionControlSettingsForm.get('username').disable({emitEvent}); - this.versionControlSettingsForm.get('password').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); - if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } - if (isNotEmptyStr(privateKeyFileName)) { - this.versionControlSettingsForm.get('privateKey').clearValidators(); - } else { - this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); - } - } - this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); - } - -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index e5e650ac85..3cf10ab168 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -15,9 +15,9 @@ limitations under the License. --> - - + + (); - hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + hasRepository$ = this.store.pipe(select(selectHasRepository)); constructor(private store: Store) { @@ -61,7 +61,7 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index 958e7b1c9b..16baa3aa89 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -225,14 +225,14 @@ const routes: Routes = [ ] }, { - path: 'vc', - component: VersionControlAdminSettingsComponent, + path: 'repository', + component: RepositoryAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], - title: 'admin.git-settings', + title: 'admin.repository-settings', breadcrumb: { - label: 'admin.git-settings', + label: 'admin.repository-settings', icon: 'manage_history' } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 51612270df..85e3d04767 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version- HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlAdminSettingsComponent + RepositoryAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html similarity index 87% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html index 869067e1fb..7b408fc93a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html @@ -15,4 +15,4 @@ limitations under the License. --> - + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts similarity index 68% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts index a203f64426..1ee0a7288a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts @@ -20,16 +20,16 @@ import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormGroup } from '@angular/forms'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; @Component({ - selector: 'tb-version-control-admin-settings', - templateUrl: './version-control-admin-settings.component.html', + selector: 'tb-repository-admin-settings', + templateUrl: './repository-admin-settings.component.html', styleUrls: [] }) -export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class RepositoryAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { - @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + @ViewChild('repositorySettingsComponent') repositorySettingsComponent: RepositorySettingsComponent; constructor(protected store: Store) { super(store); @@ -39,6 +39,6 @@ export class VersionControlAdminSettingsComponent extends PageComponent implemen } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 6297809522..ff16183509 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -135,7 +135,7 @@ export const HelpLinks = { ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', queue: helpBaseUrl + '/docs/user-guide/queue', - versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' + repositorySettings: helpBaseUrl + '/docs/user-guide/ui/repository-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 8a1b316001..298281da11 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -16,6 +16,7 @@ import { ValidatorFn } from '@angular/forms'; import { isNotEmptyStr, isNumber } from '@core/utils'; +import { VersionCreateConfig } from '@shared/models/vc.models'; export const smtpPortPattern: RegExp = /^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/; @@ -397,23 +398,29 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid return smsProviderConfiguration; } -export enum VersionControlAuthMethod { +export enum RepositoryAuthMethod { USERNAME_PASSWORD = 'USERNAME_PASSWORD', PRIVATE_KEY = 'PRIVATE_KEY' } -export const versionControlAuthMethodTranslationMap = new Map([ - [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], - [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +export const repositoryAuthMethodTranslationMap = new Map([ + [RepositoryAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [RepositoryAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] ]); -export interface EntitiesVersionControlSettings { +export interface RepositorySettings { repositoryUri: string; defaultBranch: string; - authMethod: VersionControlAuthMethod; + authMethod: RepositoryAuthMethod; username: string; password: string; privateKeyFileName: string; privateKey: string; privateKeyPassword: string; } + +export interface AutoVersionCreateConfig extends VersionCreateConfig { + branch: string; +} + +export type AutoCommitSettings = {[entityType: string]: AutoVersionCreateConfig}; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0ad52d2e0b..cd1cba93ed 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -32,6 +32,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; + saveAttributes: boolean; } export enum VersionCreateRequestType { @@ -81,6 +82,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: res[entityType] = { syncStrategy: null, saveRelations: false, + saveAttributes: false, allEntities: true, entityIds: [] }; @@ -90,6 +92,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; + loadAttributes: boolean; } export enum VersionLoadRequestType { @@ -111,6 +114,7 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { removeOtherEntities: boolean; + findExistingEntityByName: boolean; } export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { @@ -123,7 +127,9 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En for (const entityType of exportableEntityTypes) { res[entityType] = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; } return res; @@ -176,18 +182,5 @@ export interface EntityDataDiff { } export function entityExportDataToJsonString(data: EntityExportData): string { - if (!data.relations) { - data.relations = []; - } - const allKeys = new Set(); - JSON.stringify(data, (key, value) => (allKeys.add(key), value)); - return JSON.stringify(data, Array.from(allKeys).sort((key1, key2) => { - if (key1 === 'relations') { - return 1; - } else if (key2 === 'relations') { - return -1; - } else { - return 0; - } - }), 4); + return JSON.stringify(data, null, 4); } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 4a4e228cc2..8a2f63cb8f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -58,7 +58,8 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", - "restore": "Restore" + "restore": "Restore", + "confirm": "Confirm" }, "aggregation": { "aggregation": "Aggregation", @@ -321,8 +322,7 @@ "queue-submit-strategy": "Submit strategy", "queue-processing-strategy": "Processing strategy", "queue-configuration": "Queue configuration", - "git-settings": "Git settings", - "git-repository-settings": "Git repository settings", + "repository-settings": "Repository settings", "repository-url": "Repository URL", "repository-url-required": "Repository URL is required.", "default-branch": "Default branch name", @@ -338,9 +338,9 @@ "enter-passphrase": "Enter passphrase", "change-passphrase": "Change passphrase", "check-access": "Check access", - "check-vc-access-success": "Git repository access successfully verified!", - "delete-git-settings-title": "Are you sure you want to delete git settings?", - "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + "check-repository-access-success": "Repository access successfully verified!", + "delete-repository-settings-title": "Are you sure you want to delete repository settings?", + "delete-repository-settings-text": "Be careful, after the confirmation the repository settings will be removed and version control feature will be unavailable." }, "alarm": { "alarm": "Alarm", @@ -3121,6 +3121,7 @@ "version-name-required": "Version name is required", "author": "Author", "export-entity-relations": "Export entity relations", + "export-entity-attributes": "Export entity attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3133,6 +3134,7 @@ "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations", + "load-entity-attributes": "Load entity attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3151,12 +3153,15 @@ "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "load-entities-relations": "Load entities relations", + "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", + "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", "no-entities-restored": "No entities restored", "created": "{{created}} created", "updated": "{{updated}} updated", - "deleted": "{{deleted}} deleted" + "deleted": "{{deleted}} deleted", + "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm." }, "widget": { "widget-library": "Widgets Library", From 59325226ac5020227644afee6a2641bec462e8b2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 16:38:10 +0300 Subject: [PATCH 108/178] Sort order and diff fixes --- .../DefaultEntitiesVersionControlService.java | 19 +++++++++-------- .../DefaultGitVersionControlQueueService.java | 4 ++-- .../common/data/sync/ie/EntityExportData.java | 21 +++++++++++++++++++ .../thingsboard/common/util/JacksonUtil.java | 10 ++++++--- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index fb68967661..05fbef899b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -328,16 +328,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityId externalId = ((ExportableEntity) entity).getExternalId(); if (externalId == null) externalId = entityId; - EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() - .exportRelations(true) - .exportAttributes(true) - .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), - otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion), - JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { - return new EntityDataDiff(currentVersion, otherVersion, rawDiff); - }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + otherVersion -> { + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(otherVersion.getRelations() != null) + .exportAttributes(otherVersion.getAttributes() != null) + .build()); + return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), + rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7019d6adb5..095c56f143 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -109,7 +109,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson = JacksonUtil.toPrettyString(entityData); + String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -130,7 +130,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu buildCommitRequest(commit).setDeleteMsg( TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() ).build() - ).build(), wrap(commit.getFuture(), null)); + ).build(), wrap(future, null)); return future; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 542e675940..0fbdae756e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -41,6 +43,15 @@ import java.util.Map; @Data public class EntityExportData> { + public static final Comparator relationsComparator = Comparator + .comparing(EntityRelation::getFrom, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTo, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTypeGroup) + .thenComparing(EntityRelation::getType); + + public static final Comparator attrComparator = Comparator + .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + @JsonTbEntity private E entity; private EntityType entityType; @@ -48,4 +59,14 @@ public class EntityExportData> { private List relations; private Map> attributes; + public EntityExportData sort() { + if (relations != null && !relations.isEmpty()) { + relations.sort(relationsComparator); + } + if (attributes != null && !attributes.isEmpty()) { + attributes.values().forEach(list -> list.sort(attrComparator)); + } + return this; + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index b6e2b54479..d622801814 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -18,8 +18,10 @@ package org.thingsboard.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; @@ -31,8 +33,10 @@ import java.util.Arrays; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() - .enable(SerializationFeature.INDENT_OUTPUT); + public static final ObjectMapper PRETTY_SORTED_JSON_MAPPER = JsonMapper.builder().enable(SerializationFeature.INDENT_OUTPUT) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -99,7 +103,7 @@ public class JacksonUtil { public static String toPrettyString(Object o) { try { - return PRETTY_JSON_MAPPER.writeValueAsString(o); + return PRETTY_SORTED_JSON_MAPPER.writeValueAsString(o); } catch (JsonProcessingException e) { throw new RuntimeException(e); } From 7bd51cb4cec451b38e0d348f61ed046a4b781e14 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:13:50 +0300 Subject: [PATCH 109/178] Public Customer import fix. No commit when no changes --- .../importing/impl/CustomerImportService.java | 6 +++++- .../DefaultGitVersionControlQueueService.java | 18 ++++++++++-------- .../data/sync/vc/RepositorySettings.java | 1 - .../DefaultClusterVersionControlService.java | 18 +++++++++++------- .../sync/vc/DefaultGitRepositoryService.java | 9 +++++---- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index 2a7570e644..aec96b2e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -42,7 +42,11 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return customerService.saveCustomer(customer); + if (customer.isPublic()) { + return customerService.findOrCreatePublicCustomer(tenantId); + } else { + return customerService.saveCustomer(customer); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 095c56f143..c87789a096 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -152,7 +152,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch), + .setBranchName(branch), pageLink ).build()); } @@ -162,8 +162,8 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityType.name()), + .setBranchName(branch) + .setEntityType(entityType.name()), pageLink ).build()); } @@ -173,10 +173,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), pageLink ).build()); } @@ -354,7 +354,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + if (commitResponse.getTs() > 0) { + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + } commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index eb1a97ff6c..50e23a1dba 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -21,7 +21,6 @@ import lombok.Data; import java.io.Serializable; @Data -@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 6ff7347be9..b28618372e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -457,14 +457,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { - reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() - .setTs(result.getVersion().getTimestamp()) - .setCommitId(result.getVersion().getId()) - .setName(result.getVersion().getName()) - .setAuthor(result.getVersion().getAuthor()) - .setAdded(result.getAdded()) + var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded()) .setModified(result.getModified()) - .setRemoved(result.getRemoved()))); + .setRemoved(result.getRemoved()); + + if (result.getVersion() != null) { + responseBuilder.setTs(result.getVersion().getTimestamp()) + .setCommitId(result.getVersion().getId()) + .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()); + } + + reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(responseBuilder)); } private void reply(VersionControlRequestCtx ctx, Optional e) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 6db0a49d4f..99dbebc3c9 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,10 +117,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); - repository.push(commit.getWorkingBranch(), commit.getBranch()); - - result.setVersion(toVersion(gitCommit)); + if (result.getAdded() > 0 || result.getModified() > 0 || result.getRemoved() > 0) { + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); + repository.push(commit.getWorkingBranch(), commit.getBranch()); + result.setVersion(toVersion(gitCommit)); + } return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; From 1610222b72528ae7399bfc82167210f9171d5764 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 17:27:53 +0300 Subject: [PATCH 110/178] Export/import api rate limiting --- .../apiusage/DefaultRateLimitService.java | 76 +++++++++++++++++++ .../service/apiusage/RateLimitService.java | 26 +++++++ .../DefaultEntitiesExportImportService.java | 11 +++ .../DefaultTenantProfileConfiguration.java | 5 +- .../server/common/msg/tools/TbRateLimits.java | 8 +- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 777e84f348..859f3b9da2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.apiusage.RateLimitService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; @@ -55,6 +57,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); + private final RateLimitService rateLimitService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE @@ -63,6 +67,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); @@ -73,6 +81,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 15eb429805..9953f7a3aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index afcbe1b3b9..9ca13b295d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -27,6 +27,7 @@ import java.time.Duration; */ public class TbRateLimits { private final LocalBucket bucket; + private final String configuration; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); @@ -42,8 +43,7 @@ public class TbRateLimits { } else { throw new IllegalArgumentException("Failed to parse rate limits configuration: " + limitsConfiguration); } - - + this.configuration = limitsConfiguration; } public boolean tryConsume() { @@ -54,4 +54,8 @@ public class TbRateLimits { return bucket.tryConsume(number); } + public String getConfiguration() { + return configuration; + } + } From ac0b1a98794b05ff713ed45568d5eca94a1cdfa9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:46:23 +0300 Subject: [PATCH 111/178] getEntityDataInfo API call --- .../EntitiesVersionControlController.java | 13 +++++++++ .../DefaultEntitiesVersionControlService.java | 8 ++++++ .../vc/EntitiesVersionControlService.java | 3 ++ .../common/data/sync/vc/EntityDataInfo.java | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bf7011867d..36aa6fd961 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -237,6 +238,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/info/{versionId}/{entityType}/{internalEntityUuid}") + public DeferredResult getEntityDataInfo(@PathVariable String versionId, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") public DeferredResult compareEntityDataToVersion(@PathVariable String branch, @PathVariable EntityType entityType, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 05fbef899b..cca21929c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -341,6 +342,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { + return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + } + + @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { return gitServiceQueue.listBranches(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 447ae1d021..88c23616b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -62,4 +63,6 @@ public interface EntitiesVersionControlService { ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; + + ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java new file mode 100644 index 0000000000..ab548b1cfc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityDataInfo { + boolean hasRelations; + boolean hasAttributes; +} From 9eb32add2bf5ebb330d14bfd23ce74d9f30df6cd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 18:11:24 +0300 Subject: [PATCH 112/178] UI for export/import api rate limiting --- .../default-tenant-profile-configuration.component.html | 8 ++++++++ .../default-tenant-profile-configuration.component.ts | 2 ++ ui-ngx/src/app/shared/models/tenant.model.ts | 3 +++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ 4 files changed, 15 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index fef4d4cef5..08c8df06a0 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -280,4 +280,12 @@ tenant-profile.transport-device-telemetry-data-points-rate-limit + + tenant-profile.tenant-entity-export-rate-limit + + + + tenant-profile.tenant-entity-import-rate-limit + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index ddab7f3aae..6f79c34279 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -67,6 +67,8 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA transportDeviceMsgRateLimit: [null, []], transportDeviceTelemetryMsgRateLimit: [null, []], transportDeviceTelemetryDataPointsRateLimit: [null, []], + tenantEntityExportRateLimit: [null, []], + tenantEntityImportRateLimit: [null, []], maxTransportMessages: [null, [Validators.required, Validators.min(0)]], maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], maxREExecutions: [null, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 33d1367c01..ea1ff3fdc5 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -41,6 +41,9 @@ export interface DefaultTenantProfileConfiguration { transportDeviceTelemetryMsgRateLimit?: string; transportDeviceTelemetryDataPointsRateLimit?: string; + tenantEntityExportRateLimit?: string; + tenantEntityImportRateLimit?: string; + maxTransportMessages: number; maxTransportDataPoints: number; maxREExecutions: number; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index db52c04ac1..90193b1499 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2944,6 +2944,8 @@ "transport-device-msg-rate-limit": "Transport device messages rate limit.", "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "tenant-entity-export-rate-limit": "Entity version creation rate limit", + "tenant-entity-import-rate-limit": "Entity version load rate limit", "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", "max-transport-messages-required": "Maximum number of transport messages is required.", "max-transport-messages-range": "Maximum number of transport messages can't be negative", From 7841c5ff6fa27ccbc80544d2ef222a63a0b3d19e Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 18:42:33 +0300 Subject: [PATCH 113/178] UI: Implement auto-commit settings --- .../server/controller/AdminController.java | 8 +- ui-ngx/src/app/core/http/admin.service.ts | 2 +- ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/components/home-components.module.ts | 7 +- .../vc/auto-commit-settings.component.html | 125 +++++++++++ .../vc/auto-commit-settings.component.scss | 74 ++++++ .../vc/auto-commit-settings.component.ts | 210 ++++++++++++++++++ ...entity-types-version-create.component.html | 4 +- .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 4 +- .../vc/entity-types-version-load.component.ts | 2 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../auto-commit-admin-settings.component.html | 23 ++ .../auto-commit-admin-settings.component.ts | 51 +++++ .../vc/branch-autocomplete.component.html | 4 +- .../vc/branch-autocomplete.component.ts | 3 + ui-ngx/src/app/shared/models/constants.ts | 3 +- .../assets/locale/locale.constant-en_US.json | 16 +- 19 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index ed740f73fb..470ec738af 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -273,7 +273,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings") + @GetMapping("/autoCommitSettings") @ResponseBody public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { try { @@ -287,7 +287,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings/exists") + @GetMapping("/autoCommitSettings/exists") @ResponseBody public Boolean autoCommitSettingsExists() throws ThingsboardException { try { @@ -301,7 +301,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vc/autoCommitSettings") + @PostMapping("/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); return autoCommitSettingsService.save(getTenantId(), settings); @@ -311,7 +311,7 @@ public class AdminController extends BaseController { notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) public void deleteAutoCommitSettings() throws ThingsboardException { try { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 38f18bd598..9b8c3b76b9 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -101,7 +101,7 @@ export class AdminService { return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); } - private autoCommitSettingsExists(config?: RequestConfig): Observable { + public autoCommitSettingsExists(config?: RequestConfig): Observable { return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 50938e5869..af62bf7918 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -369,7 +369,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '120px', + height: '160px', icon: 'settings', pages: [ { @@ -392,6 +392,13 @@ export class MenuService { type: 'link', path: '/settings/repository', icon: 'manage_history' + }, + { + id: guid(), + name: 'admin.auto-commit-settings', + type: 'link', + path: '/settings/auto-commit', + icon: 'settings_backup_restore' } ] } @@ -541,6 +548,11 @@ export class MenuService { name: 'admin.repository-settings', icon: 'manage_history', path: '/settings/repository', + }, + { + name: 'admin.auto-commit-settings', + icon: 'settings_backup_restore', + path: '/settings/auto-commit' } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 613f236cf6..153a8d9759 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -164,6 +164,7 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; +import { AutoCommitSettingsComponent } from '@home/components/vc/auto-commit-settings.component'; @NgModule({ declarations: @@ -298,7 +299,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], imports: [ CommonModule, @@ -427,7 +429,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html new file mode 100644 index 0000000000..3e668b54e6 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -0,0 +1,125 @@ + +
+ + +
+ admin.auto-commit-settings + +
+
+ + + +
+ + +
+ admin.auto-commit-entities +
+
+
+ + +
+ +
+
+
+ + + +
+ + +
+ +
+
+ + +
+ + +
+
+
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
+
+
+ + +
+
+
+ admin.no-auto-commit-entities-prompt +
+
+ + + +
+
+
+
+ + + +
+ + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss new file mode 100644 index 0000000000..ea8017b9f1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.auto-commit-settings { + margin: 8px; + .mat-divider { + position: relative; + } + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + tb-branch-autocomplete { + min-width: 200px; + max-width: 200px; + display: block; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts new file mode 100644 index 0000000000..8ac301338e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { AbstractControl, FormArray, FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { AutoCommitSettings, AutoVersionCreateConfig } from '@shared/models/settings.models'; +import { TranslateService } from '@ngx-translate/core'; +import { DialogService } from '@core/services/dialog.service'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-auto-commit-settings', + templateUrl: './auto-commit-settings.component.html', + styleUrls: ['./auto-commit-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class AutoCommitSettingsComponent extends PageComponent implements OnInit { + + autoCommitSettingsForm: FormGroup; + settings: AutoCommitSettings = null; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private sanitizer: DomSanitizer, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.autoCommitSettingsForm = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.adminService.autoCommitSettingsExists().pipe( + catchError(() => of(false)), + mergeMap((hasAutoCommitSettings) => { + if (hasAutoCommitSettings) { + return this.adminService.getAutoCommitSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(settings), {emitEvent: false}); + }); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.autoCommitSettingsForm.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.autoCommitSettingsForm.get('entityTypes') as FormArray).removeAt(index); + this.autoCommitSettingsForm.markAsDirty(); + } + + public addEnabled(): boolean { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + const config: AutoVersionCreateConfig = { + branch: null, + saveRelations: false, + saveAttributes: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + public removeAll() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + entityTypeText(entityTypeControl: AbstractControl): SafeHtml { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: AutoVersionCreateConfig = entityTypeControl.get('config').value; + let message = entityType ? this.translate.instant(entityTypeTranslations.get(entityType).typePlural) : 'Undefined'; + let branchName; + if (config.branch) { + branchName = config.branch; + } else { + branchName = this.translate.instant('version-control.default'); + } + message += ` (${this.translate.instant('version-control.auto-commit-to-branch', {branch: branchName})})`; + return this.sanitizer.bypassSecurityTrustHtml(message); + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + save(): void { + const value: [{entityType: string, config: AutoVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const settings: AutoCommitSettings = {}; + if (value && value.length) { + value.forEach((val) => { + settings[val.entityType] = val.config; + }); + } + this.adminService.saveAutoCommitSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(savedSettings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-auto-commit-settings-title', ), + this.translate.instant('admin.delete-auto-commit-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteAutoCommitSettings().subscribe( + () => { + this.settings = null; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(this.settings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + }); + } + + private prepareEntityTypesFormArray(settings: AutoCommitSettings | null): FormArray { + const entityTypesControls: Array = []; + if (settings) { + for (const entityType of Object.keys(settings)) { + const config = settings[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: AutoVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + branch: [config.branch, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []] + }) + } + ); + return entityTypeControl; + } + + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 3c1dd73117..d4da91028f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -17,7 +17,7 @@ -->
- version-control.entity-types + version-control.entities-to-export
version-control.no-entity-types + class="tb-prompt">version-control.no-entities-to-export-prompt
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 2ff8c03227..cc5ddcb568 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -41,6 +41,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Input() + entityName: string; + @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; @@ -61,7 +64,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ngOnInit(): void { this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], - versionName: [null, [Validators.required]], + versionName: [this.translate.instant('version-control.default-create-entity-version-name', + {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], saveAttributes: [false, []] }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts index 6036cf35a0..c72cbc0848 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -308,7 +308,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: this.versionName, @@ -322,6 +322,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, } } }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; } } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..dd851e9f23 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -15,27 +15,28 @@ limitations under the License. --> -
- +
+

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

 + *ngIf="entityDataInfo && (isLoading$ | async)"> -
+ +
- + {{ 'version-control.load-entity-relations' | translate }} - + {{ 'version-control.load-entity-attributes' | translate }}
-
+
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index d4da91028f..0ab5eca4e0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -66,10 +66,10 @@
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index d5a08ee141..c07fa138be 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -65,10 +65,10 @@
- {{ 'version-control.load-entities-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entities-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index b5f2fb6937..fa2855a6ed 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -39,10 +39,10 @@ - {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..0d97823c21 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,10 +27,10 @@
- {{ 'version-control.load-entity-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entity-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 40d427df24..2f428f11fe 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3125,8 +3125,8 @@ "version-name": "Version name", "version-name-required": "Version name is required", "author": "Author", - "export-entity-relations": "Export entity relations", - "export-entity-attributes": "Export entity attributes", + "export-relations": "Export relations", + "export-attributes": "Export attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3138,8 +3138,8 @@ "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", - "load-entity-relations": "Load entity relations", - "load-entity-attributes": "Load entity attributes", + "load-relations": "Load relations", + "load-attributes": "Load attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3159,8 +3159,6 @@ "add-entity-type": "Add entity type", "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", - "load-entities-relations": "Load entities relations", - "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", From 8382850d1a0c056561e6b071477f5b273a9cdee8 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 16:35:20 +0300 Subject: [PATCH 116/178] UI: Minor improvements --- .../vc/auto-commit-settings.component.html | 6 +++--- .../vc/auto-commit-settings.component.ts | 4 ++-- .../vc/complex-version-create.component.html | 3 ++- .../vc/complex-version-create.component.ts | 4 +++- .../entity-types-version-create.component.html | 6 +++--- .../vc/entity-types-version-create.component.ts | 4 ++-- .../vc/entity-types-version-load.component.html | 6 +++--- .../vc/entity-types-version-load.component.ts | 6 +++--- .../vc/entity-version-create.component.html | 6 +++--- .../vc/entity-version-create.component.ts | 2 +- .../vc/entity-version-restore.component.html | 6 +++--- .../vc/entity-version-restore.component.ts | 4 ++-- .../app/shared/components/popover.component.ts | 12 +++++++----- .../app/shared/components/popover.service.ts | 8 ++------ ui-ngx/src/app/shared/models/vc.models.ts | 17 ++++++++++++----- .../assets/locale/locale.constant-en_US.json | 4 +++- 16 files changed, 54 insertions(+), 44 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index a33d9ea374..e95cfc59fb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -75,12 +75,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index 8ac301338e..dc009d49cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -97,8 +97,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; const config: AutoVersionCreateConfig = { branch: null, - saveRelations: false, - saveAttributes: false + saveAttributes: true, + saveRelations: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 2c2f5bf274..4ace5e8849 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -39,13 +39,14 @@ {{ 'version-control.version-name-required' | translate }} - + version-control.default-sync-strategy {{syncStrategyTranslations.get(strategy) | translate}} + diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts index c152e6b447..c5c4124627 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -20,7 +20,7 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { ComplexVersionCreateRequest, createDefaultEntityTypesVersionCreate, - SyncStrategy, syncStrategyTranslationMap, + SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -51,6 +51,8 @@ export class ComplexVersionCreateComponent extends PageComponent implements OnIn syncStrategyTranslations = syncStrategyTranslationMap; + syncStrategyHints = syncStrategyHintMap; + resultMessage: string; versionCreateResult: VersionCreationResult = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 0ab5eca4e0..90712727e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -65,12 +65,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 44cb11dfa1..0d80b71adb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -183,8 +183,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionCreateConfig = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index c07fa138be..4c1cd26f84 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -64,12 +64,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 14586a5ee6..d983920cec 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -164,10 +164,10 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On public addEntityType() { const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 72dc6e7d48..90f338ad0e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -38,12 +38,12 @@ {{ 'version-control.version-name-required' | translate }} - - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index cc5ddcb568..c2c6fa48b5 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -67,7 +67,7 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni versionName: [this.translate.instant('version-control.default-create-entity-version-name', {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], - saveAttributes: [false, []] + saveAttributes: [true, []] }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 6a39e272e3..8dc81e38e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,12 +27,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 5e3a63243d..146a8570c2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -70,8 +70,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [true, []], - loadAttributes: [true, []] + loadAttributes: [true, []], + loadRelations: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; diff --git a/ui-ngx/src/app/shared/components/popover.component.ts b/ui-ngx/src/app/shared/components/popover.component.ts index c7e498a236..99321974af 100644 --- a/ui-ngx/src/app/shared/components/popover.component.ts +++ b/ui-ngx/src/app/shared/components/popover.component.ts @@ -55,7 +55,7 @@ import { POSITION_MAP, PropertyMapping } from '@shared/components/popover.models'; -import { distinctUntilChanged, takeUntil } from 'rxjs/operators'; +import { distinctUntilChanged, take, takeUntil } from 'rxjs/operators'; import { isNotEmptyStr, onParentScrollOrWindowResize } from '@core/utils'; export type TbPopoverTrigger = 'click' | 'focus' | 'hover' | null; @@ -372,7 +372,7 @@ export class TbPopoverComponent implements OnDestroy, OnInit { } get tbVisible(): boolean { - return this.visible; + return this.visible && this.tbAnimationState === 'active'; } visible = false; @@ -514,10 +514,12 @@ export class TbPopoverComponent implements OnDestroy, OnInit { const el = this.origin.elementRef.nativeElement; this.intersectionObserver.unobserve(el); } - - this.tbVisible = false; - this.tbVisibleChange.next(false); + this.tbAnimationState = 'void'; this.cdr.detectChanges(); + this.tbAnimationDone.pipe(take(1)).subscribe(() => { + this.tbVisible = false; + this.cdr.detectChanges(); + }); } updateByDirective(): void { diff --git a/ui-ngx/src/app/shared/components/popover.service.ts b/ui-ngx/src/app/shared/components/popover.service.ts index 2b64ee8009..713e8d337b 100644 --- a/ui-ngx/src/app/shared/components/popover.service.ts +++ b/ui-ngx/src/app/shared/components/popover.service.ts @@ -80,9 +80,7 @@ export class TbPopoverService { component.tbShowCloseButton = showCloseButton; component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { @@ -132,9 +130,7 @@ export class TbPopoverService { component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { visibleFn(false); - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index e3a7be5c54..ee718c3aec 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -64,6 +64,13 @@ export const syncStrategyTranslationMap = new Map( ] ); +export const syncStrategyHintMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge-hint'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite-hint'] + ] +); + export interface EntityTypeVersionCreateConfig extends VersionCreateConfig { syncStrategy: SyncStrategy; entityIds: string[]; @@ -81,8 +88,8 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: for (const entityType of exportableEntityTypes) { res[entityType] = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; @@ -126,10 +133,10 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; for (const entityType of exportableEntityTypes) { res[entityType] = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; } return res; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2bc8d6fdbe..ad9ba856ab 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3168,7 +3168,9 @@ "deleted": "{{deleted}} deleted", "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm.", "auto-commit-to-branch": "auto-commit to {{ branch }} branch", - "default-create-entity-version-name": "{{entityName}} update" + "default-create-entity-version-name": "{{entityName}} update", + "sync-strategy-merge-hint": "Sync strategy merge hint", + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" }, "widget": { "widget-library": "Widgets Library", From 54e388433479d7e8259bb05d5151fe85196a6c3c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 1 Jun 2022 16:55:38 +0300 Subject: [PATCH 117/178] Version load error and 'exportCredentials' setting support --- .../EntitiesVersionControlController.java | 3 +- .../impl/BaseEntityExportService.java | 4 +- .../exporting/impl/DeviceExportService.java | 7 +- .../impl/RuleChainExportService.java | 3 +- .../ie/importing/impl/AssetImportService.java | 3 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 3 +- .../impl/DashboardImportService.java | 2 +- .../importing/impl/DeviceImportService.java | 5 +- .../impl/DeviceProfileImportService.java | 3 +- .../impl/ImportServiceException.java | 20 ++ .../impl/MissingEntityException.java | 30 +++ .../impl/RuleChainImportService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 251 ++++++++++-------- .../DefaultGitVersionControlQueueService.java | 6 +- .../vc/EntitiesVersionControlService.java | 5 +- .../service/sync/vc/LoadEntityException.java | 32 +++ .../common/data/sync/ie/EntityExportData.java | 6 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../common/data/sync/vc/EntityDataInfo.java | 1 + .../common/data/sync/vc/EntityLoadError.java | 42 +++ .../data/sync/vc/EntityTypeLoadResult.java | 33 +++ .../data/sync/vc/VersionLoadResult.java | 29 +- .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 26 files changed, 368 insertions(+), 138 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 58c2659c4a..d55fe3dec0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -296,7 +297,7 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 80850cb462..9e14531e48 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -30,11 +30,11 @@ public abstract class BaseEntityExportService exportData, IdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index caa1e20ad7..ea938c89d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -64,7 +64,8 @@ import java.util.stream.Collectors; @Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { - @Autowired @Lazy + @Autowired + @Lazy private ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; @@ -94,7 +95,7 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -108,7 +109,7 @@ public abstract class BaseEntityImportService importResult, D exportData, @@ -210,7 +211,8 @@ public abstract class BaseEntityImportService() { @Override - public void onSuccess(@Nullable Void unused) {} + public void onSuccess(@Nullable Void unused) { + } @Override public void onFailure(Throwable thr) { @@ -246,7 +248,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) - .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); + .orElseThrow(() -> new MissingEntityException(externalId)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index aec96b2e47..2b2e2d1ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,7 +42,7 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { if (customer.isPublic()) { return customerService.findOrCreatePublicCustomer(tenantId); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 16ebdd580e..a67a536ded 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -64,7 +64,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { JsonNode configuration = dashboard.getConfiguration(); String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> { return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index eb123a63af..9a3730bba9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,11 +42,11 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java new file mode 100644 index 0000000000..1e869429d8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +public class ImportServiceException extends RuntimeException{ + private static final long serialVersionUID = -4932715239522125041L; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java new file mode 100644 index 0000000000..a0a961bfef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +public class MissingEntityException extends ImportServiceException { + + private static final long serialVersionUID = 3669135386955906022L; + @Getter + private final EntityId entityId; + + public MissingEntityException(EntityId entityId) { + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 1f1f15a74e..cd29be99d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -62,7 +62,7 @@ public class RuleChainImportService extends BaseEntityImportService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 735dc33a24..8aa4915fa8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,8 +22,10 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -45,10 +47,12 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityLoadError; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -64,12 +68,14 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -83,8 +89,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -172,6 +180,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -203,123 +212,157 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return Futures.transform(future, entityData -> { - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(false) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); - }, executor); + return Futures.transform(future, entityData -> doInTemplate(status -> loadSingleEntity(user, config, entityData)), executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return executor.submit(() -> transactionTemplate.execute(status -> { - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - List saveReferencesCallbacks = new ArrayList<>(); - List sendEventsCallbacks = new ArrayList<>(); - - versionLoadRequest.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()) - .forEach(entityType -> { - EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); + return executor.submit(() -> doInTemplate(status -> loadMultipleEntities(user, versionLoadRequest))); + } + default: + throw new IllegalArgumentException("Unsupported version load request"); + } + } - try { - int limit = 100; - int offset = 0; - List entityDataList; - do { - entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); - for (EntityExportData entityData : entityDataList) { - EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - offset += limit; - importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); - } while (entityDataList.size() == limit); - } catch (Exception e) { - throw new RuntimeException(e); - } - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created(created.get()) - .updated(updated.get()) - .build()); - }); + private VersionLoadResult doInTemplate(TransactionCallback result) { + try { + return transactionTemplate.execute(result); + } catch (LoadEntityException e) { + return onError(e.getData(), e.getCause()); + } + } - versionLoadRequest.getEntityTypes().keySet().stream() - .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) - .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) - .forEach(entityType -> { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - - sendEventsCallbacks.add(() -> { - entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), - entity, null, ActionType.DELETED, null, user); - }); - VersionLoadResult result = results.get(entityType); - result.setDeleted(result.getDeleted() + 1); - } - }); - }); + private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, + EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(false) + .build(), true, true); + return VersionLoadResult.success(EntityTypeLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + } - for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { - try { - saveReferencesCallback.run(); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } + private VersionLoadResult loadMultipleEntities(SecurityUser user, EntityTypeVersionLoadRequest versionLoadRequest) { + Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + + List entityTypes = versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + for (EntityType entityType : entityTypes) { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), versionLoadRequest.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); } - for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + results.put(entityType, EntityTypeLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); + } + + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); + EntityTypeLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); } - } - return new ArrayList<>(results.values()); - })); + }); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return VersionLoadResult.success(new ArrayList<>(results.values())); + } + + private VersionLoadResult onError(EntityExportData entityData, Throwable e) { + return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e)); + } + + private Optional analyze(Throwable e, EntityExportData entityData) { + if (e == null) { + return Optional.empty(); + } else { + if (e instanceof DeviceCredentialsValidationException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + } else if (e instanceof MissingEntityException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + } else { + return analyze(e.getCause(), entityData); } - default: - throw new IllegalArgumentException("Unsupported version load request"); } } @@ -347,7 +390,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), - entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null, false), MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index e168312206..759bc83acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -108,7 +108,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { SettableFuture future = SettableFuture.create(); - String path = getRelativePath(entityData.getEntityType(), getExternalId(entityData.getEntity())); + String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( @@ -120,10 +120,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return future; } - private EntityId getExternalId(ExportableEntity entity) { - return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); - } - @Override public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { SettableFuture future = SettableFuture.create(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 3b38182075..0a0f56eac6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -23,11 +23,12 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -49,7 +50,7 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java new file mode 100644 index 0000000000..a1b036d37b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@SuppressWarnings("rawtypes") +public class LoadEntityException extends RuntimeException { + + private static final long serialVersionUID = -1749719992370409504L; + @Getter + private final EntityExportData data; + + public LoadEntityException(EntityExportData data, Throwable cause) { + super(cause); + this.data = data; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 1109d9ef09..d715ec9611 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.ie; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -75,4 +76,9 @@ public class EntityExportData> { return this; } + @JsonIgnore + public EntityId getExternalId() { + return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0800a1f7c1..1a625640e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityExportSettings { private boolean exportRelations; private boolean exportAttributes; + private boolean exportCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 564b0134e3..9b95b8eca2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -28,4 +28,5 @@ public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java index ab548b1cfc..2529ec8af6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -25,4 +25,5 @@ import lombok.NoArgsConstructor; public class EntityDataInfo { boolean hasRelations; boolean hasAttributes; + boolean hasCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java new file mode 100644 index 0000000000..900a05e470 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EntityLoadError { + + private String type; + private EntityId source; + private EntityId target; + + public static EntityLoadError credentialsError(EntityId sourceId) { + return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build(); + } + + public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) { + return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java new file mode 100644 index 0000000000..cc0a8a642a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityTypeLoadResult { + private EntityType entityType; + private int created; + private int updated; + private int deleted; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index 0039546fb0..736de26079 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -15,19 +15,30 @@ */ package org.thingsboard.server.common.data.sync.vc; -import lombok.AllArgsConstructor; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.EntityType; + +import java.util.List; @Data -@AllArgsConstructor -@NoArgsConstructor @Builder +@JsonInclude(JsonInclude.Include.NON_NULL) public class VersionLoadResult { - private EntityType entityType; - private int created; - private int updated; - private int deleted; + + private List result; + private EntityLoadError error; + + public static VersionLoadResult success(List result) { + return VersionLoadResult.builder().result(result).build(); + } + + public static VersionLoadResult success(EntityTypeLoadResult result) { + return VersionLoadResult.builder().result(List.of(result)).build(); + } + + public static VersionLoadResult error(EntityLoadError error) { + return VersionLoadResult.builder().error(error).build(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 4426717fe9..86154bdfa0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -25,4 +25,5 @@ public class VersionCreateConfig implements Serializable { private boolean saveRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 2d68a2b7e3..a27cf06538 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -22,5 +22,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; + private boolean loadCredentials; } From 14023e4cc9fc1a957d8c8a9f81e06d14ea19aed0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 18:02:52 +0300 Subject: [PATCH 118/178] UI: Implement rule chain page version control. --- .../vc/entity-version-create.component.ts | 44 +++++++----- .../vc/entity-versions-table.component.html | 7 +- .../vc/entity-versions-table.component.ts | 9 ++- .../vc/version-control.component.html | 2 + .../vc/version-control.component.ts | 7 ++ .../rulechain/rulechain-page.component.html | 11 +++ .../rulechain/rulechain-page.component.scss | 15 ++++ .../rulechain/rulechain-page.component.ts | 68 +++++++++++++++++-- .../shared/components/breadcrumb.component.ts | 12 +++- 9 files changed, 148 insertions(+), 27 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index c2c6fa48b5..108d4aa081 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -27,6 +27,7 @@ import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; @Component({ selector: 'tb-entity-version-create', @@ -50,6 +51,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() onContentUpdated: () => void; + @Input() + onBeforeCreateVersion: () => Observable; + createVersionFormGroup: FormGroup; resultMessage: string; @@ -78,25 +82,29 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni } export(): void { - const request: SingleEntityVersionCreateRequest = { - entityId: this.entityId, - branch: this.createVersionFormGroup.get('branch').value, - versionName: this.createVersionFormGroup.get('versionName').value, - config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value, - saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (!result.added && !result.modified) { - this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); - if (this.onContentUpdated) { - this.onContentUpdated(); + const before = this.onBeforeCreateVersion ? this.onBeforeCreateVersion() : of(null); + before.subscribe(() => { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + config: { + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { + this.onClose(result, request.branch); } - } else if (this.onClose) { - this.onClose(result, request.branch); - } + }); }); } } + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index d2dbc119bf..79da474200 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -16,7 +16,7 @@ -->
-
+
@@ -45,6 +45,11 @@ update {{'version-control.create-entities-version' | translate }} +
+ + {{ 'version-control.export-credentials' | translate }} + {{ 'version-control.export-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index dc009d49cd..fd100f6247 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -39,6 +39,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit autoCommitSettingsForm: FormGroup; settings: AutoCommitSettings = null; + entityTypes = EntityType; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -98,7 +100,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const config: AutoVersionCreateConfig = { branch: null, saveAttributes: true, - saveRelations: false + saveRelations: false, + saveCredentials: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -199,7 +202,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit config: this.fb.group({ branch: [config.branch, []], saveRelations: [config.saveRelations, []], - saveAttributes: [config.saveAttributes, []] + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 4ace5e8849..9e15076788 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -70,7 +70,7 @@
-
{{ resultMessage }}
+
-
-
+
+
{{ 'version-control.no-entities-restored' | translate }}
-
{{ versionLoadResultMessage(versionLoadResult) }}
+
+
{{ entityTypeLoadResultMessage(entityTypeLoadResult) }}
-
{{ resultMessage }}
+
{{ resultMessage }}
- -
+
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + + +
+
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 146a8570c2..c31030f74b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -30,11 +30,12 @@ import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { delay } from 'rxjs/operators'; +import { SafeHtml } from '@angular/platform-browser'; @Component({ selector: 'tb-entity-version-restore', templateUrl: './entity-version-restore.component.html', - styleUrls: [] + styleUrls: ['./version-control.scss'] }) export class EntityVersionRestoreComponent extends PageComponent implements OnInit { @@ -51,7 +52,7 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: EntityId; @Input() - onClose: (result: Array | null) => void; + onClose: (result: VersionLoadResult | null) => void; @Input() popoverComponent: TbPopoverComponent; @@ -60,6 +61,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn restoreFormGroup: FormGroup; + errorMessage: SafeHtml; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -71,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], - loadRelations: [true, []] + loadRelations: [true, []], + loadCredentials: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -95,13 +99,22 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: this.externalEntityId, config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, - loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false + loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { - this.onClose(result); + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else { + if (this.onClose) { + this.onClose(result); + } } }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 79da474200..b6accc40dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -15,8 +15,8 @@ limitations under the License. --> -
-
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index ec5cb02256..3256c3c956 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -20,6 +20,13 @@ height: 100%; display: block; .tb-entity-table { + + &.tb-popover-mode { + position: relative; + width: 800px; + height: 600px; + } + .tb-entity-table-content { width: 100%; height: 100%; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 38b0a33da4..d776939aaf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -49,6 +49,7 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -63,7 +64,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni singleEntityMode = false; @Input() - popoverMode = false; + popoverComponent: TbPopoverComponent; @Input() onBeforeCreateVersion: () => Observable; @@ -211,14 +212,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - createVersionPopover.updatePosition(); - setTimeout(() => { - createVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + createVersionPopover.tbComponentRef.instance.popoverComponent = createVersionPopover; } } @@ -243,14 +239,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - complexCreateVersionPopover.updatePosition(); - setTimeout(() => { - complexCreateVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + complexCreateVersionPopover.tbComponentRef.instance.popoverComponent = complexCreateVersionPopover; } } @@ -293,9 +284,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni versionName: entityVersion.name, versionId: entityVersion.id, externalEntityId: this.externalEntityIdValue, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreVersionPopover.hide(); - if (result && result.length) { + if (result && !result.error && result.result.length) { this.versionRestored.emit(); } } @@ -318,10 +309,11 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni branch: this.branch, versionName: entityVersion.name, versionId: entityVersion.id, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreEntitiesVersionPopover.hide(); } }, {}, {}, {}, false); + restoreEntitiesVersionPopover.tbComponentRef.instance.popoverComponent = restoreEntitiesVersionPopover; } } diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index bdab6c061b..3e8bd271f3 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.repository-settings diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts index c73cf0e632..1120185db9 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit } from '@angular/core'; +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; import { select, Store } from '@ngrx/store'; @@ -33,6 +33,7 @@ import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; import { selectHasRepository } from '@core/auth/auth.selectors'; import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-repository-settings', @@ -44,6 +45,9 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit @Input() detailsMode = false; + @Input() + popoverComponent: TbPopoverComponent; + repositorySettingsForm: FormGroup; settings: RepositorySettings = null; @@ -61,6 +65,7 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit private adminService: AdminService, private dialogService: DialogService, private translate: TranslateService, + private cd: ChangeDetectorRef, public fb: FormBuilder) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 0ede98554d..eaf80fd664 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -16,11 +16,12 @@ --> { this.reloadRuleChain(); }); diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index ee718c3aec..0c6a667b88 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -33,6 +33,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; + saveCredentials: boolean; } export enum VersionCreateRequestType { @@ -90,6 +91,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: true, saveRelations: true, + saveCredentials: true, allEntities: true, entityIds: [] }; @@ -100,6 +102,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; + loadCredentials: boolean; } export enum VersionLoadRequestType { @@ -135,6 +138,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En res[entityType] = { loadAttributes: true, loadRelations: true, + loadCredentials: true, removeOtherEntities: false, findExistingEntityByName: true }; @@ -161,13 +165,36 @@ export interface VersionCreationResult { removed: number; } -export interface VersionLoadResult { +export interface EntityTypeLoadResult { entityType: EntityType; created: number; updated: number; deleted: number; } +export enum EntityLoadErrorType { + DEVICE_CREDENTIALS_CONFLICT = 'DEVICE_CREDENTIALS_CONFLICT', + MISSING_REFERENCED_ENTITY = 'MISSING_REFERENCED_ENTITY' +} + +export const entityLoadErrorTranslationMap = new Map( + [ + [EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT, 'version-control.device-credentials-conflict'], + [EntityLoadErrorType.MISSING_REFERENCED_ENTITY, 'version-control.missing-referenced-entity'] + ] +); + +export interface EntityLoadError { + type: EntityLoadErrorType; + source: EntityId; + target: EntityId; +} + +export interface VersionLoadResult { + result: Array; + error: EntityLoadError; +} + export interface EntityExportData> { entity: E; entityType: EntityType; @@ -195,4 +222,5 @@ export function entityExportDataToJsonString(data: EntityExportData): strin export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; + hasCredentials: boolean; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ad9ba856ab..8a4d8ff4db 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3127,6 +3127,7 @@ "author": "Author", "export-relations": "Export relations", "export-attributes": "Export attributes", + "export-credentials": "Export credentials", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3140,6 +3141,7 @@ "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-relations": "Load relations", "load-attributes": "Load attributes", + "load-credentials": "Load credentials", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3158,7 +3160,7 @@ "no-entities-to-restore-prompt": "Please specify entities to restore", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.
{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.
{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", @@ -3170,7 +3172,9 @@ "auto-commit-to-branch": "auto-commit to {{ branch }} branch", "default-create-entity-version-name": "{{entityName}} update", "sync-strategy-merge-hint": "Sync strategy merge hint", - "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint", + "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", + "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}." }, "widget": { "widget-library": "Widgets Library", From da224c9651f6ac14993776704b82aed6a2b319ee Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 2 Jun 2022 14:10:56 +0300 Subject: [PATCH 123/178] UI: Implement dashboard version control --- .../dashboard-page.component.html | 7 +++ .../dashboard-page.component.ts | 60 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index dd7212cf5b..553971005f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -81,6 +81,13 @@ (click)="isFullscreen = !isFullscreen"> {{ isFullscreen ? 'fullscreen_exit' : 'fullscreen' }} +